|
a |
|
b/qiita_db/archive.py |
|
|
1 |
# ----------------------------------------------------------------------------- |
|
|
2 |
# Copyright (c) 2014--, The Qiita Development Team. |
|
|
3 |
# |
|
|
4 |
# Distributed under the terms of the BSD 3-clause License. |
|
|
5 |
# |
|
|
6 |
# The full license is in the file LICENSE, distributed with this software. |
|
|
7 |
# ----------------------------------------------------------------------------- |
|
|
8 |
import qiita_db as qdb |
|
|
9 |
|
|
|
10 |
|
|
|
11 |
class Archive(qdb.base.QiitaObject): |
|
|
12 |
r"""Extra information for any features stored in a BIOM Artifact |
|
|
13 |
|
|
|
14 |
Methods |
|
|
15 |
------- |
|
|
16 |
insert_from_artifact |
|
|
17 |
get_merging_scheme_from_job |
|
|
18 |
retrieve_feature_values |
|
|
19 |
insert_features |
|
|
20 |
|
|
|
21 |
See Also |
|
|
22 |
-------- |
|
|
23 |
qiita_db.QiitaObject |
|
|
24 |
""" |
|
|
25 |
|
|
|
26 |
@classmethod |
|
|
27 |
def merging_schemes(cls): |
|
|
28 |
r"""Returns the available merging schemes |
|
|
29 |
|
|
|
30 |
Returns |
|
|
31 |
------- |
|
|
32 |
Iterator |
|
|
33 |
Iterator over the sample ids |
|
|
34 |
|
|
|
35 |
See Also |
|
|
36 |
-------- |
|
|
37 |
keys |
|
|
38 |
""" |
|
|
39 |
with qdb.sql_connection.TRN: |
|
|
40 |
sql = """SELECT archive_merging_scheme_id, archive_merging_scheme |
|
|
41 |
FROM qiita.archive_merging_scheme""" |
|
|
42 |
qdb.sql_connection.TRN.add(sql) |
|
|
43 |
return dict(qdb.sql_connection.TRN.execute_fetchindex()) |
|
|
44 |
|
|
|
45 |
@classmethod |
|
|
46 |
def _inserting_main_steps(cls, ms, features): |
|
|
47 |
with qdb.sql_connection.TRN: |
|
|
48 |
sql = """INSERT INTO qiita.archive_merging_scheme |
|
|
49 |
(archive_merging_scheme) |
|
|
50 |
SELECT %s WHERE NOT EXISTS ( |
|
|
51 |
SELECT 1 FROM qiita.archive_merging_scheme |
|
|
52 |
WHERE archive_merging_scheme = %s)""" |
|
|
53 |
qdb.sql_connection.TRN.add(sql, [ms, ms]) |
|
|
54 |
sql = """SELECT archive_merging_scheme_id |
|
|
55 |
FROM qiita.archive_merging_scheme |
|
|
56 |
WHERE archive_merging_scheme = %s""" |
|
|
57 |
qdb.sql_connection.TRN.add(sql, [ms]) |
|
|
58 |
amsi = qdb.sql_connection.TRN.execute_fetchlast() |
|
|
59 |
|
|
|
60 |
vals = [[amsi, _id, val] for _id, val in features.items()] |
|
|
61 |
qdb.sql_connection.TRN.add( |
|
|
62 |
"SELECT archive_upsert(%s, %s, %s)", vals, many=True) |
|
|
63 |
qdb.sql_connection.TRN.execute() |
|
|
64 |
|
|
|
65 |
@classmethod |
|
|
66 |
def insert_from_artifact(cls, artifact, features): |
|
|
67 |
r"""Inserts new features to the database based on a given artifact |
|
|
68 |
|
|
|
69 |
Parameters |
|
|
70 |
---------- |
|
|
71 |
artifact : qiita_db.artifact.Artifact |
|
|
72 |
The artifact from which the features were generated |
|
|
73 |
features : dict {str: str} |
|
|
74 |
A dictionary of the features and the values to be stored |
|
|
75 |
|
|
|
76 |
Raises |
|
|
77 |
------ |
|
|
78 |
ValueError |
|
|
79 |
If the Artifact type is not BIOM |
|
|
80 |
If the artifact doesn't have a biom filepath |
|
|
81 |
""" |
|
|
82 |
with qdb.sql_connection.TRN: |
|
|
83 |
atype = artifact.artifact_type |
|
|
84 |
if atype != 'BIOM': |
|
|
85 |
raise ValueError( |
|
|
86 |
"To archive artifact must be BIOM but %s" % atype) |
|
|
87 |
|
|
|
88 |
bfps = [x['fp'] for x in artifact.filepaths |
|
|
89 |
if x['fp_type'] == 'biom'] |
|
|
90 |
if not bfps: |
|
|
91 |
raise ValueError("The artifact has no biom files") |
|
|
92 |
|
|
|
93 |
# [0] as it returns a list |
|
|
94 |
ms = qdb.util.get_artifacts_information( |
|
|
95 |
[artifact.id])[0]['algorithm'] |
|
|
96 |
|
|
|
97 |
cls._inserting_main_steps(ms, features) |
|
|
98 |
|
|
|
99 |
@classmethod |
|
|
100 |
def get_merging_scheme_from_job(cls, job): |
|
|
101 |
r"""Inserts new features to the database based on a given job |
|
|
102 |
|
|
|
103 |
Parameters |
|
|
104 |
---------- |
|
|
105 |
job : qiita_db.processing_job.ProcessingJob |
|
|
106 |
The Qiita process job_id generating the artifact holding the |
|
|
107 |
features to be retrieved or stored. |
|
|
108 |
|
|
|
109 |
Raises |
|
|
110 |
------ |
|
|
111 |
ValueError |
|
|
112 |
If the Artifact type is not BIOM |
|
|
113 |
If the artifact doesn't have a biom filepath |
|
|
114 |
""" |
|
|
115 |
with qdb.sql_connection.TRN: |
|
|
116 |
acmd = job.command |
|
|
117 |
parent = job.input_artifacts[0] |
|
|
118 |
parent_pparameters = parent.processing_parameters |
|
|
119 |
phms = None |
|
|
120 |
if parent_pparameters is None: |
|
|
121 |
parent_cmd_name = None |
|
|
122 |
parent_parameters = None |
|
|
123 |
parent_merging_scheme = None |
|
|
124 |
else: |
|
|
125 |
pcmd = parent_pparameters.command |
|
|
126 |
parent_cmd_name = pcmd.name |
|
|
127 |
parent_parameters = parent_pparameters.values |
|
|
128 |
parent_merging_scheme = pcmd.merging_scheme |
|
|
129 |
if not parent_merging_scheme['ignore_parent_command']: |
|
|
130 |
gp = parent.parents[0] |
|
|
131 |
gp_params = gp.processing_parameters |
|
|
132 |
if gp_params is not None: |
|
|
133 |
gp_cmd = gp_params.command |
|
|
134 |
phms = qdb.util.human_merging_scheme( |
|
|
135 |
parent_cmd_name, parent_merging_scheme, |
|
|
136 |
gp_cmd.name, gp_cmd.merging_scheme, |
|
|
137 |
parent_parameters, [], gp_params.values) |
|
|
138 |
|
|
|
139 |
hms = qdb.util.human_merging_scheme( |
|
|
140 |
acmd.name, acmd.merging_scheme, |
|
|
141 |
parent_cmd_name, parent_merging_scheme, |
|
|
142 |
job.parameters.values, [], parent_parameters) |
|
|
143 |
|
|
|
144 |
if phms is not None: |
|
|
145 |
hms = qdb.util.merge_overlapping_strings(hms, phms) |
|
|
146 |
|
|
|
147 |
return hms |
|
|
148 |
|
|
|
149 |
@classmethod |
|
|
150 |
def retrieve_feature_values(cls, archive_merging_scheme=None, |
|
|
151 |
features=None): |
|
|
152 |
r"""Retrieves all features/values from the archive |
|
|
153 |
|
|
|
154 |
Parameters |
|
|
155 |
---------- |
|
|
156 |
archive_merging_scheme : optional, str |
|
|
157 |
The name of the archive_merging_scheme to retrieve |
|
|
158 |
features : list of str, optional |
|
|
159 |
List of features to retrieve information from the archive |
|
|
160 |
|
|
|
161 |
Notes |
|
|
162 |
----- |
|
|
163 |
If archive_merging_scheme is None it will return all |
|
|
164 |
feature values |
|
|
165 |
""" |
|
|
166 |
with qdb.sql_connection.TRN: |
|
|
167 |
extras = [] |
|
|
168 |
vals = [] |
|
|
169 |
if archive_merging_scheme is not None: |
|
|
170 |
extras.append("""archive_merging_scheme = %s""") |
|
|
171 |
vals.append(archive_merging_scheme) |
|
|
172 |
if features is not None: |
|
|
173 |
extras.append("""archive_feature IN %s""") |
|
|
174 |
# depending on the method calling test retrieve_feature_values |
|
|
175 |
# the features elements can be string or bytes; making sure |
|
|
176 |
# everything is string for SQL |
|
|
177 |
vals.append( |
|
|
178 |
tuple([f.decode('ascii') if isinstance(f, bytes) else f |
|
|
179 |
for f in features])) |
|
|
180 |
|
|
|
181 |
sql = """SELECT archive_feature, archive_feature_value |
|
|
182 |
FROM qiita.archive_feature_value |
|
|
183 |
LEFT JOIN qiita.archive_merging_scheme |
|
|
184 |
USING (archive_merging_scheme_id) {0} |
|
|
185 |
ORDER BY archive_merging_scheme, archive_feature""" |
|
|
186 |
|
|
|
187 |
if extras: |
|
|
188 |
sql = sql.format('WHERE ' + ' AND '.join(extras)) |
|
|
189 |
qdb.sql_connection.TRN.add(sql, vals) |
|
|
190 |
else: |
|
|
191 |
qdb.sql_connection.TRN.add(sql.format('')) |
|
|
192 |
|
|
|
193 |
return dict(qdb.sql_connection.TRN.execute_fetchindex()) |
|
|
194 |
|
|
|
195 |
@classmethod |
|
|
196 |
def insert_features(cls, merging_scheme, features): |
|
|
197 |
r"""Inserts new features to the database based on a given artifact |
|
|
198 |
|
|
|
199 |
Parameters |
|
|
200 |
---------- |
|
|
201 |
merging_scheme : str |
|
|
202 |
The merging scheme to store these features |
|
|
203 |
features : dict {str: str} |
|
|
204 |
A dictionary of the features and the values to be stored |
|
|
205 |
|
|
|
206 |
Returns |
|
|
207 |
------- |
|
|
208 |
dict, feature: value |
|
|
209 |
The inserted new values |
|
|
210 |
""" |
|
|
211 |
cls._inserting_main_steps(merging_scheme, features) |
|
|
212 |
|
|
|
213 |
inserted = cls.retrieve_feature_values( |
|
|
214 |
archive_merging_scheme=merging_scheme, features=features.keys()) |
|
|
215 |
|
|
|
216 |
return inserted |