Datasets
Models
Diff of
/qiita_db/software.py
[000000]
..
[879b32]
Switch to unified view
| a | b/qiita_db/software.py | ||
|---|---|---|---|
| 1 | # ----------------------------------------------------------------------------- |
||
| 2 | # Copyright (c) 2014--, The Qiita Development Team. |
||
| 3 | # |
||
| 4 | # Distributed under the terms of the BSD 3-clause License. |
||
| 5 | # |
||
| 6 | # The full license is in the file LICENSE, distributed with this software. |
||
| 7 | # ----------------------------------------------------------------------------- |
||
| 8 | |||
| 9 | from json import dumps, loads |
||
| 10 | from copy import deepcopy |
||
| 11 | import inspect |
||
| 12 | import warnings |
||
| 13 | |||
| 14 | import networkx as nx |
||
| 15 | |||
| 16 | from qiita_core.qiita_settings import qiita_config |
||
| 17 | import qiita_db as qdb |
||
| 18 | |||
| 19 | from configparser import ConfigParser |
||
| 20 | |||
| 21 | |||
| 22 | class Command(qdb.base.QiitaObject): |
||
| 23 | r"""An executable command available in the system |
||
| 24 | |||
| 25 | Attributes |
||
| 26 | ---------- |
||
| 27 | active |
||
| 28 | post_processing_cmd |
||
| 29 | analysis_only |
||
| 30 | default_parameter_sets |
||
| 31 | description |
||
| 32 | merging_scheme |
||
| 33 | name |
||
| 34 | naming_order |
||
| 35 | optional_parameters |
||
| 36 | outputs |
||
| 37 | parameters |
||
| 38 | required_parameters |
||
| 39 | software |
||
| 40 | description |
||
| 41 | cli |
||
| 42 | parameters_table |
||
| 43 | |||
| 44 | Methods |
||
| 45 | ------- |
||
| 46 | _check_id |
||
| 47 | activate |
||
| 48 | |||
| 49 | Class Methods |
||
| 50 | ------------- |
||
| 51 | create |
||
| 52 | exists |
||
| 53 | get_commands_by_input_type |
||
| 54 | get_html_generator |
||
| 55 | get_validator |
||
| 56 | |||
| 57 | See Also |
||
| 58 | -------- |
||
| 59 | qiita_db.software.Software |
||
| 60 | """ |
||
| 61 | _table = "software_command" |
||
| 62 | |||
| 63 | @classmethod |
||
| 64 | def get_commands_by_input_type(cls, artifact_types, active_only=True, |
||
| 65 | exclude_analysis=True, prep_type=None): |
||
| 66 | """Returns the commands that can process the given artifact types |
||
| 67 | |||
| 68 | Parameters |
||
| 69 | ---------- |
||
| 70 | artifact_type : list of str |
||
| 71 | The artifact types |
||
| 72 | active_only : bool, optional |
||
| 73 | If True, return only active commands, otherwise return all commands |
||
| 74 | Default: True |
||
| 75 | exclude_analysis : bool, optional |
||
| 76 | If True, return commands that are not part of the analysis pipeline |
||
| 77 | |||
| 78 | Returns |
||
| 79 | ------- |
||
| 80 | generator of qiita_db.software.Command |
||
| 81 | The commands that can process the given artifact tyoes |
||
| 82 | """ |
||
| 83 | with qdb.sql_connection.TRN: |
||
| 84 | sql = """SELECT DISTINCT command_id |
||
| 85 | FROM qiita.command_parameter |
||
| 86 | JOIN qiita.parameter_artifact_type |
||
| 87 | USING (command_parameter_id) |
||
| 88 | JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 89 | JOIN qiita.software_command USING (command_id) |
||
| 90 | WHERE artifact_type IN %s""" |
||
| 91 | if active_only: |
||
| 92 | sql += " AND active = True" |
||
| 93 | if exclude_analysis: |
||
| 94 | sql += " AND is_analysis = False" |
||
| 95 | qdb.sql_connection.TRN.add(sql, [tuple(artifact_types)]) |
||
| 96 | cids = set(qdb.sql_connection.TRN.execute_fetchflatten()) |
||
| 97 | |||
| 98 | if prep_type is not None: |
||
| 99 | dws = [w for w in qdb.software.DefaultWorkflow.iter() |
||
| 100 | if prep_type in w.data_type] |
||
| 101 | if dws: |
||
| 102 | cmds = {n.default_parameter.command.id |
||
| 103 | for w in dws for n in w.graph.nodes} |
||
| 104 | cids = cmds & cids |
||
| 105 | |||
| 106 | return [cls(cid) for cid in cids] |
||
| 107 | |||
| 108 | @classmethod |
||
| 109 | def get_html_generator(cls, artifact_type): |
||
| 110 | """Returns the command that generete the HTML for the given artifact |
||
| 111 | |||
| 112 | Parameters |
||
| 113 | ---------- |
||
| 114 | artifact_type : str |
||
| 115 | The artifact type to search the HTML generator for |
||
| 116 | |||
| 117 | Returns |
||
| 118 | ------- |
||
| 119 | qiita_db.software.Command |
||
| 120 | The newly created command |
||
| 121 | |||
| 122 | Raises |
||
| 123 | ------ |
||
| 124 | qdb.exceptions.QiitaDBError when the generete the HTML command can't |
||
| 125 | be found |
||
| 126 | """ |
||
| 127 | with qdb.sql_connection.TRN: |
||
| 128 | sql = """SELECT command_id |
||
| 129 | FROM qiita.software_command |
||
| 130 | JOIN qiita.software_artifact_type USING (software_id) |
||
| 131 | JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 132 | WHERE artifact_type = %s |
||
| 133 | AND name = 'Generate HTML summary' |
||
| 134 | AND active = true""" |
||
| 135 | qdb.sql_connection.TRN.add(sql, [artifact_type]) |
||
| 136 | try: |
||
| 137 | res = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 138 | except IndexError: |
||
| 139 | raise qdb.exceptions.QiitaDBError( |
||
| 140 | "There is no command to generate the HTML summary for " |
||
| 141 | "artifact type '%s'" % artifact_type) |
||
| 142 | |||
| 143 | return cls(res) |
||
| 144 | |||
| 145 | @classmethod |
||
| 146 | def get_validator(cls, artifact_type): |
||
| 147 | """Returns the command that validates the given artifact |
||
| 148 | |||
| 149 | Parameters |
||
| 150 | ---------- |
||
| 151 | artifact_type : str |
||
| 152 | The artifact type to search the Validate for |
||
| 153 | |||
| 154 | Returns |
||
| 155 | ------- |
||
| 156 | qiita_db.software.Command |
||
| 157 | The newly created command |
||
| 158 | |||
| 159 | Raises |
||
| 160 | ------ |
||
| 161 | qdb.exceptions.QiitaDBError when the Validate command can't be found |
||
| 162 | """ |
||
| 163 | with qdb.sql_connection.TRN: |
||
| 164 | sql = """SELECT command_id |
||
| 165 | FROM qiita.software_command |
||
| 166 | JOIN qiita.software_artifact_type USING (software_id) |
||
| 167 | JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 168 | WHERE artifact_type = %s |
||
| 169 | AND name = 'Validate' |
||
| 170 | AND active = true""" |
||
| 171 | qdb.sql_connection.TRN.add(sql, [artifact_type]) |
||
| 172 | try: |
||
| 173 | res = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 174 | except IndexError: |
||
| 175 | raise qdb.exceptions.QiitaDBError( |
||
| 176 | "There is no command to generate the Validate for " |
||
| 177 | "artifact type '%s'" % artifact_type) |
||
| 178 | |||
| 179 | return cls(res) |
||
| 180 | |||
| 181 | def _check_id(self, id_): |
||
| 182 | """Check that the provided ID actually exists in the database |
||
| 183 | |||
| 184 | Parameters |
||
| 185 | ---------- |
||
| 186 | id_ : int |
||
| 187 | The ID to test |
||
| 188 | |||
| 189 | Notes |
||
| 190 | ----- |
||
| 191 | This function overwrites the base function, as the sql layout doesn't |
||
| 192 | follow the same conventions done in the other classes. |
||
| 193 | """ |
||
| 194 | with qdb.sql_connection.TRN: |
||
| 195 | sql = """SELECT EXISTS( |
||
| 196 | SELECT * |
||
| 197 | FROM qiita.software_command |
||
| 198 | WHERE command_id = %s)""" |
||
| 199 | qdb.sql_connection.TRN.add(sql, [id_]) |
||
| 200 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 201 | |||
| 202 | @classmethod |
||
| 203 | def exists(cls, software, name): |
||
| 204 | """Checks if the command already exists in the system |
||
| 205 | |||
| 206 | Parameters |
||
| 207 | ---------- |
||
| 208 | qiita_db.software.Software |
||
| 209 | The software to which this command belongs to. |
||
| 210 | name : str |
||
| 211 | The name of the command |
||
| 212 | |||
| 213 | Returns |
||
| 214 | ------- |
||
| 215 | bool |
||
| 216 | Whether the command exists in the system or not |
||
| 217 | """ |
||
| 218 | with qdb.sql_connection.TRN: |
||
| 219 | sql = """SELECT EXISTS(SELECT * |
||
| 220 | FROM qiita.software_command |
||
| 221 | WHERE software_id = %s |
||
| 222 | AND name = %s)""" |
||
| 223 | qdb.sql_connection.TRN.add(sql, [software.id, name]) |
||
| 224 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 225 | |||
| 226 | @classmethod |
||
| 227 | def create(cls, software, name, description, parameters, outputs=None, |
||
| 228 | analysis_only=False): |
||
| 229 | r"""Creates a new command in the system |
||
| 230 | |||
| 231 | The supported types for the parameters are: |
||
| 232 | - string: the parameter is a free text input |
||
| 233 | - integer: the parameter is an integer |
||
| 234 | - float: the parameter is a float |
||
| 235 | - artifact: the parameter is an artifact instance, the artifact id |
||
| 236 | will be stored |
||
| 237 | - reference: the parameter is a reference instance, the reference |
||
| 238 | id will be stored |
||
| 239 | - choice: the format of this should be `choice:<json-dump-of-list>` |
||
| 240 | in which json-dump-of-list is the JSON dump of a list containing |
||
| 241 | the acceptable values |
||
| 242 | |||
| 243 | Parameters |
||
| 244 | ---------- |
||
| 245 | software : qiita_db.software.Software |
||
| 246 | The software to which this command belongs to. |
||
| 247 | name : str |
||
| 248 | The name of the command |
||
| 249 | description : str |
||
| 250 | The description of the command |
||
| 251 | parameters : dict |
||
| 252 | The description of the parameters that this command received. The |
||
| 253 | format is: {parameter_name: (parameter_type, default, name_order, |
||
| 254 | check_biom_merge, qiita_optional_parameter (optional))}, |
||
| 255 | where parameter_name, parameter_type and default are strings, |
||
| 256 | name_order is an optional integer value and check_biom_merge is |
||
| 257 | an optional boolean value. name_order is used to specify the order |
||
| 258 | of the parameter when automatically naming the artifacts. |
||
| 259 | check_biom_merge is used when merging artifacts in the analysis |
||
| 260 | pipeline. qiita_optional_parameter is an optional bool to "force" |
||
| 261 | the parameter to be optional |
||
| 262 | outputs : dict, optional |
||
| 263 | The description of the outputs that this command generated. The |
||
| 264 | format is either {output_name: artifact_type} or |
||
| 265 | {output_name: (artifact_type, check_biom_merge)} |
||
| 266 | analysis_only : bool, optional |
||
| 267 | If true, then the command will only be available on the analysis |
||
| 268 | pipeline. Default: False. |
||
| 269 | |||
| 270 | Returns |
||
| 271 | ------- |
||
| 272 | qiita_db.software.Command |
||
| 273 | The newly created command |
||
| 274 | |||
| 275 | Raises |
||
| 276 | ------ |
||
| 277 | QiitaDBError |
||
| 278 | - If parameters is empty |
||
| 279 | - If the parameters dictionary is malformed |
||
| 280 | - If one of the parameter types is not supported |
||
| 281 | - If the default value of a choice parameter is not listed in |
||
| 282 | the available choices |
||
| 283 | QiitaDBDuplicateError |
||
| 284 | - If the command already exists |
||
| 285 | |||
| 286 | Notes |
||
| 287 | ----- |
||
| 288 | If the default value for a parameter is NULL, then the parameter will |
||
| 289 | be required. On the other hand, if it is provided, the parameter will |
||
| 290 | be optional and the default value will be used when the user doesn't |
||
| 291 | overwrite it. |
||
| 292 | """ |
||
| 293 | # Perform some sanity checks in the parameters dictionary |
||
| 294 | if not parameters: |
||
| 295 | raise qdb.exceptions.QiitaDBError( |
||
| 296 | "Error creating command %s. At least one parameter should " |
||
| 297 | "be provided." % name) |
||
| 298 | sql_param_values = [] |
||
| 299 | sql_artifact_params = [] |
||
| 300 | for pname, vals in parameters.items(): |
||
| 301 | qiita_optional_parameter = False |
||
| 302 | if 'qiita_optional_parameter' in vals: |
||
| 303 | qiita_optional_parameter = True |
||
| 304 | vals.remove('qiita_optional_parameter') |
||
| 305 | lenvals = len(vals) |
||
| 306 | if lenvals == 2: |
||
| 307 | ptype, dflt = vals |
||
| 308 | name_order = None |
||
| 309 | check_biom_merge = False |
||
| 310 | elif lenvals == 4: |
||
| 311 | ptype, dflt, name_order, check_biom_merge = vals |
||
| 312 | else: |
||
| 313 | raise qdb.exceptions.QiitaDBError( |
||
| 314 | "Malformed parameters dictionary, the format should be " |
||
| 315 | "either {param_name: [parameter_type, default]} or " |
||
| 316 | "{parameter_name: (parameter_type, default, name_order, " |
||
| 317 | "check_biom_merge)}. Found: %s for parameter name %s" |
||
| 318 | % (vals, pname)) |
||
| 319 | |||
| 320 | # Check that the type is one of the supported types |
||
| 321 | supported_types = ['string', 'integer', 'float', 'reference', |
||
| 322 | 'boolean', 'prep_template', 'analysis'] |
||
| 323 | if ptype not in supported_types and not ptype.startswith( |
||
| 324 | ('choice', 'mchoice', 'artifact')): |
||
| 325 | supported_types.extend(['choice', 'mchoice', 'artifact']) |
||
| 326 | raise qdb.exceptions.QiitaDBError( |
||
| 327 | "Unsupported parameters type '%s' for parameter %s. " |
||
| 328 | "Supported types are: %s" |
||
| 329 | % (ptype, pname, ', '.join(supported_types))) |
||
| 330 | |||
| 331 | if ptype.startswith(('choice', 'mchoice')) and dflt is not None: |
||
| 332 | choices = set(loads(ptype.split(':')[1])) |
||
| 333 | dflt_val = dflt |
||
| 334 | if ptype.startswith('choice'): |
||
| 335 | # In the choice case, the dflt value is a single string, |
||
| 336 | # create a list with it the string on it to use the |
||
| 337 | # issuperset call below |
||
| 338 | dflt_val = [dflt_val] |
||
| 339 | else: |
||
| 340 | # jsonize the list to store it in the DB |
||
| 341 | dflt = dumps(dflt) |
||
| 342 | if not choices.issuperset(dflt_val): |
||
| 343 | raise qdb.exceptions.QiitaDBError( |
||
| 344 | "The default value '%s' for the parameter %s is not " |
||
| 345 | "listed in the available choices: %s" |
||
| 346 | % (dflt, pname, ', '.join(choices))) |
||
| 347 | |||
| 348 | if ptype.startswith('artifact'): |
||
| 349 | atypes = loads(ptype.split(':')[1]) |
||
| 350 | sql_artifact_params.append( |
||
| 351 | [pname, 'artifact', atypes]) |
||
| 352 | else: |
||
| 353 | # a parameter will be required (not optional) if |
||
| 354 | # qiita_optional_parameter is false and there is the default |
||
| 355 | # value (dflt) is None |
||
| 356 | required = not qiita_optional_parameter and dflt is None |
||
| 357 | sql_param_values.append([pname, ptype, required, dflt, |
||
| 358 | name_order, check_biom_merge]) |
||
| 359 | |||
| 360 | with qdb.sql_connection.TRN: |
||
| 361 | if cls.exists(software, name): |
||
| 362 | raise qdb.exceptions.QiitaDBDuplicateError( |
||
| 363 | "command", "software: %d, name: %s" |
||
| 364 | % (software.id, name)) |
||
| 365 | # Add the command to the DB |
||
| 366 | sql = """INSERT INTO qiita.software_command |
||
| 367 | (name, software_id, description, is_analysis) |
||
| 368 | VALUES (%s, %s, %s, %s) |
||
| 369 | RETURNING command_id""" |
||
| 370 | sql_params = [name, software.id, description, analysis_only] |
||
| 371 | qdb.sql_connection.TRN.add(sql, sql_params) |
||
| 372 | c_id = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 373 | |||
| 374 | # Add the parameters to the DB |
||
| 375 | sql = """INSERT INTO qiita.command_parameter |
||
| 376 | (command_id, parameter_name, parameter_type, |
||
| 377 | required, default_value, name_order, check_biom_merge) |
||
| 378 | VALUES (%s, %s, %s, %s, %s, %s, %s) |
||
| 379 | RETURNING command_parameter_id""" |
||
| 380 | sql_params = [ |
||
| 381 | [c_id, pname, p_type, reqd, default, no, chm] |
||
| 382 | for pname, p_type, reqd, default, no, chm in sql_param_values] |
||
| 383 | qdb.sql_connection.TRN.add(sql, sql_params, many=True) |
||
| 384 | qdb.sql_connection.TRN.execute() |
||
| 385 | |||
| 386 | # Add the artifact parameters |
||
| 387 | sql_type = """INSERT INTO qiita.parameter_artifact_type |
||
| 388 | (command_parameter_id, artifact_type_id) |
||
| 389 | VALUES (%s, %s)""" |
||
| 390 | supported_types = [] |
||
| 391 | for pname, p_type, atypes in sql_artifact_params: |
||
| 392 | sql_params = [c_id, pname, p_type, True, None, None, False] |
||
| 393 | qdb.sql_connection.TRN.add(sql, sql_params) |
||
| 394 | pid = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 395 | sql_params = [ |
||
| 396 | [pid, qdb.util.convert_to_id(at, 'artifact_type')] |
||
| 397 | for at in atypes] |
||
| 398 | qdb.sql_connection.TRN.add(sql_type, sql_params, many=True) |
||
| 399 | supported_types.extend([atid for _, atid in sql_params]) |
||
| 400 | |||
| 401 | # If the software type is 'artifact definition', there are a couple |
||
| 402 | # of extra steps |
||
| 403 | if software.type == 'artifact definition': |
||
| 404 | # If supported types is not empty, link the software with these |
||
| 405 | # types |
||
| 406 | if supported_types: |
||
| 407 | sql = """INSERT INTO qiita.software_artifact_type |
||
| 408 | (software_id, artifact_type_id) |
||
| 409 | VALUES (%s, %s)""" |
||
| 410 | sql_params = [[software.id, atid] |
||
| 411 | for atid in supported_types] |
||
| 412 | qdb.sql_connection.TRN.add(sql, sql_params, many=True) |
||
| 413 | # If this is the validate command, we need to add the |
||
| 414 | # provenance and name parameters. These are used internally, |
||
| 415 | # that's why we are adding them here |
||
| 416 | if name == 'Validate': |
||
| 417 | sql = """INSERT INTO qiita.command_parameter |
||
| 418 | (command_id, parameter_name, parameter_type, |
||
| 419 | required, default_value) |
||
| 420 | VALUES (%s, 'name', 'string', 'False', |
||
| 421 | 'dflt_name'), |
||
| 422 | (%s, 'provenance', 'string', 'False', NULL) |
||
| 423 | """ |
||
| 424 | qdb.sql_connection.TRN.add(sql, [c_id, c_id]) |
||
| 425 | |||
| 426 | # Add the outputs to the command |
||
| 427 | if outputs: |
||
| 428 | sql_args = [] |
||
| 429 | for pname, at in outputs.items(): |
||
| 430 | if isinstance(at, tuple): |
||
| 431 | sql_args.append( |
||
| 432 | [pname, c_id, |
||
| 433 | qdb.util.convert_to_id(at[0], 'artifact_type'), |
||
| 434 | at[1]]) |
||
| 435 | else: |
||
| 436 | try: |
||
| 437 | at_id = qdb.util.convert_to_id(at, 'artifact_type') |
||
| 438 | except qdb.exceptions.QiitaDBLookupError: |
||
| 439 | msg = (f'Error creating {software.name}, {name}, ' |
||
| 440 | f'{description} - Unknown artifact_type: ' |
||
| 441 | f'{at}') |
||
| 442 | raise ValueError(msg) |
||
| 443 | sql_args.append([pname, c_id, at_id, False]) |
||
| 444 | |||
| 445 | sql = """INSERT INTO qiita.command_output |
||
| 446 | (name, command_id, artifact_type_id, |
||
| 447 | check_biom_merge) |
||
| 448 | VALUES (%s, %s, %s, %s)""" |
||
| 449 | qdb.sql_connection.TRN.add(sql, sql_args, many=True) |
||
| 450 | qdb.sql_connection.TRN.execute() |
||
| 451 | |||
| 452 | return cls(c_id) |
||
| 453 | |||
| 454 | @property |
||
| 455 | def software(self): |
||
| 456 | """The software to which this command belongs to |
||
| 457 | |||
| 458 | Returns |
||
| 459 | ------- |
||
| 460 | qiita_db.software.Software |
||
| 461 | the software to which this command belongs to |
||
| 462 | """ |
||
| 463 | with qdb.sql_connection.TRN: |
||
| 464 | sql = """SELECT software_id |
||
| 465 | FROM qiita.software_command |
||
| 466 | WHERE command_id = %s""" |
||
| 467 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 468 | return Software(qdb.sql_connection.TRN.execute_fetchlast()) |
||
| 469 | |||
| 470 | @property |
||
| 471 | def name(self): |
||
| 472 | """The name of the command |
||
| 473 | |||
| 474 | Returns |
||
| 475 | ------- |
||
| 476 | str |
||
| 477 | The name of the command |
||
| 478 | """ |
||
| 479 | with qdb.sql_connection.TRN: |
||
| 480 | sql = """SELECT name |
||
| 481 | FROM qiita.software_command |
||
| 482 | WHERE command_id = %s""" |
||
| 483 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 484 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 485 | |||
| 486 | @property |
||
| 487 | def post_processing_cmd(self): |
||
| 488 | """Additional processing commands required for merging |
||
| 489 | |||
| 490 | Returns |
||
| 491 | ------- |
||
| 492 | str |
||
| 493 | Returns the additional processing command for merging |
||
| 494 | """ |
||
| 495 | with qdb.sql_connection.TRN: |
||
| 496 | sql = """SELECT post_processing_cmd |
||
| 497 | FROM qiita.software_command |
||
| 498 | WHERE command_id = %s""" |
||
| 499 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 500 | |||
| 501 | cmd = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 502 | if cmd: |
||
| 503 | # assume correctly formatted json data |
||
| 504 | # load data into dictionary; don't return JSON |
||
| 505 | return loads(qdb.sql_connection.TRN.execute_fetchlast()) |
||
| 506 | |||
| 507 | return None |
||
| 508 | |||
| 509 | @property |
||
| 510 | def description(self): |
||
| 511 | """The description of the command |
||
| 512 | |||
| 513 | Returns |
||
| 514 | ------- |
||
| 515 | str |
||
| 516 | The description of the command |
||
| 517 | """ |
||
| 518 | with qdb.sql_connection.TRN: |
||
| 519 | sql = """SELECT description |
||
| 520 | FROM qiita.software_command |
||
| 521 | WHERE command_id = %s""" |
||
| 522 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 523 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 524 | |||
| 525 | @property |
||
| 526 | def parameters(self): |
||
| 527 | """Returns the parameters that the command accepts |
||
| 528 | |||
| 529 | Returns |
||
| 530 | ------- |
||
| 531 | dict |
||
| 532 | Dictionary of {parameter_name: [ptype, dflt]} |
||
| 533 | """ |
||
| 534 | with qdb.sql_connection.TRN: |
||
| 535 | sql = """SELECT parameter_name, parameter_type, default_value |
||
| 536 | FROM qiita.command_parameter |
||
| 537 | WHERE command_id = %s""" |
||
| 538 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 539 | res = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 540 | return {pname: [ptype, dflt] for pname, ptype, dflt in res} |
||
| 541 | |||
| 542 | @property |
||
| 543 | def required_parameters(self): |
||
| 544 | """Returns the required parameters that the command accepts |
||
| 545 | |||
| 546 | Returns |
||
| 547 | ------- |
||
| 548 | dict |
||
| 549 | Dictionary of {parameter_name: ptype} |
||
| 550 | """ |
||
| 551 | with qdb.sql_connection.TRN: |
||
| 552 | sql = """SELECT command_parameter_id, parameter_name, |
||
| 553 | parameter_type, array_agg( |
||
| 554 | artifact_type ORDER BY artifact_type) AS |
||
| 555 | artifact_type |
||
| 556 | FROM qiita.command_parameter |
||
| 557 | LEFT JOIN qiita.parameter_artifact_type |
||
| 558 | USING (command_parameter_id) |
||
| 559 | LEFT JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 560 | WHERE command_id = %s AND required = True |
||
| 561 | GROUP BY command_parameter_id""" |
||
| 562 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 563 | res = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 564 | return {pname: (ptype, atype) for _, pname, ptype, atype in res} |
||
| 565 | |||
| 566 | @property |
||
| 567 | def optional_parameters(self): |
||
| 568 | """Returns the optional parameters that the command accepts |
||
| 569 | |||
| 570 | Returns |
||
| 571 | ------- |
||
| 572 | dict |
||
| 573 | Dictionary of {parameter_name: [ptype, default]} |
||
| 574 | """ |
||
| 575 | with qdb.sql_connection.TRN: |
||
| 576 | sql = """SELECT parameter_name, parameter_type, default_value |
||
| 577 | FROM qiita.command_parameter |
||
| 578 | WHERE command_id = %s AND required = false""" |
||
| 579 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 580 | res = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 581 | |||
| 582 | # Define a function to load the json storing the default parameters |
||
| 583 | # if ptype is multiple choice. When I added it to the for loop as |
||
| 584 | # a one liner if, made the code a bit hard to read |
||
| 585 | def dflt_fmt(dflt, ptype): |
||
| 586 | if ptype.startswith('mchoice'): |
||
| 587 | return loads(dflt) |
||
| 588 | return dflt |
||
| 589 | |||
| 590 | return {pname: [ptype, dflt_fmt(dflt, ptype)] |
||
| 591 | for pname, ptype, dflt in res} |
||
| 592 | |||
| 593 | @property |
||
| 594 | def default_parameter_sets(self): |
||
| 595 | """Returns the list of default parameter sets |
||
| 596 | |||
| 597 | Returns |
||
| 598 | ------- |
||
| 599 | generator |
||
| 600 | generator of qiita_db.software.DefaultParameters |
||
| 601 | """ |
||
| 602 | with qdb.sql_connection.TRN: |
||
| 603 | sql = """SELECT default_parameter_set_id |
||
| 604 | FROM qiita.default_parameter_set |
||
| 605 | WHERE command_id = %s |
||
| 606 | ORDER BY default_parameter_set_id""" |
||
| 607 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 608 | res = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 609 | for pid in res: |
||
| 610 | yield DefaultParameters(pid) |
||
| 611 | |||
| 612 | @property |
||
| 613 | def outputs(self): |
||
| 614 | """Returns the list of output artifact types |
||
| 615 | |||
| 616 | Returns |
||
| 617 | ------- |
||
| 618 | list of str |
||
| 619 | The output artifact types |
||
| 620 | """ |
||
| 621 | with qdb.sql_connection.TRN: |
||
| 622 | sql = """SELECT name, artifact_type |
||
| 623 | FROM qiita.command_output |
||
| 624 | JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 625 | WHERE command_id = %s""" |
||
| 626 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 627 | return qdb.sql_connection.TRN.execute_fetchindex() |
||
| 628 | |||
| 629 | @property |
||
| 630 | def active(self): |
||
| 631 | """Returns if the command is active or not |
||
| 632 | |||
| 633 | Returns |
||
| 634 | ------- |
||
| 635 | bool |
||
| 636 | Whether the command is active or not |
||
| 637 | |||
| 638 | Notes |
||
| 639 | ----- |
||
| 640 | This method differentiates between commands based on analysis_only or |
||
| 641 | the software type. The commands that are not for analysis (processing) |
||
| 642 | and are from an artifact definition software will return as active |
||
| 643 | if they have the same name than a command that is active; this helps |
||
| 644 | for situations where the processing plugins are updated but some |
||
| 645 | commands didn't change its version. |
||
| 646 | """ |
||
| 647 | with qdb.sql_connection.TRN: |
||
| 648 | cmd_type = self.software.type |
||
| 649 | if self.analysis_only or cmd_type == 'artifact definition': |
||
| 650 | sql = """SELECT active |
||
| 651 | FROM qiita.software_command |
||
| 652 | WHERE command_id = %s""" |
||
| 653 | else: |
||
| 654 | sql = """SELECT EXISTS ( |
||
| 655 | SELECT active FROM qiita.software_command |
||
| 656 | WHERE name IN ( |
||
| 657 | SELECT name FROM qiita.software_command |
||
| 658 | WHERE command_id = %s) AND active = true)""" |
||
| 659 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 660 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 661 | |||
| 662 | def activate(self): |
||
| 663 | """Activates the command""" |
||
| 664 | sql = """UPDATE qiita.software_command |
||
| 665 | SET active = %s |
||
| 666 | WHERE command_id = %s""" |
||
| 667 | qdb.sql_connection.perform_as_transaction(sql, [True, self.id]) |
||
| 668 | |||
| 669 | @property |
||
| 670 | def analysis_only(self): |
||
| 671 | """Returns if the command is an analysis-only command |
||
| 672 | |||
| 673 | Returns |
||
| 674 | ------- |
||
| 675 | bool |
||
| 676 | Whether the command is analysis only or not |
||
| 677 | """ |
||
| 678 | with qdb.sql_connection.TRN: |
||
| 679 | sql = """SELECT is_analysis |
||
| 680 | FROM qiita.software_command |
||
| 681 | WHERE command_id = %s""" |
||
| 682 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 683 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 684 | |||
| 685 | @property |
||
| 686 | def naming_order(self): |
||
| 687 | """The ordered list of parameters to use to name the output artifacts |
||
| 688 | |||
| 689 | Returns |
||
| 690 | ------- |
||
| 691 | list of str |
||
| 692 | """ |
||
| 693 | with qdb.sql_connection.TRN: |
||
| 694 | sql = """SELECT parameter_name |
||
| 695 | FROM qiita.command_parameter |
||
| 696 | WHERE command_id = %s AND name_order IS NOT NULL |
||
| 697 | ORDER BY name_order""" |
||
| 698 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 699 | return qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 700 | |||
| 701 | @property |
||
| 702 | def merging_scheme(self): |
||
| 703 | """The values to check when merging the output result |
||
| 704 | |||
| 705 | Returns |
||
| 706 | ------- |
||
| 707 | dict of {'parameters': [list of str], |
||
| 708 | 'outputs': [list of str] |
||
| 709 | 'ignore_parent_command': bool} |
||
| 710 | """ |
||
| 711 | with qdb.sql_connection.TRN: |
||
| 712 | sql = """SELECT parameter_name |
||
| 713 | FROM qiita.command_parameter |
||
| 714 | WHERE command_id = %s AND check_biom_merge = TRUE |
||
| 715 | ORDER BY parameter_name""" |
||
| 716 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 717 | params = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 718 | sql = """SELECT name |
||
| 719 | FROM qiita.command_output |
||
| 720 | WHERE command_id = %s AND check_biom_merge = TRUE |
||
| 721 | ORDER BY name""" |
||
| 722 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 723 | outputs = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 724 | |||
| 725 | sql = """SELECT ignore_parent_command |
||
| 726 | FROM qiita.software_command |
||
| 727 | WHERE command_id = %s""" |
||
| 728 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 729 | ipc = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 730 | |||
| 731 | return {'parameters': params, |
||
| 732 | 'outputs': outputs, |
||
| 733 | 'ignore_parent_command': ipc} |
||
| 734 | |||
| 735 | @property |
||
| 736 | def resource_allocation(self): |
||
| 737 | """The resource allocation defined in the database for this command |
||
| 738 | |||
| 739 | Returns |
||
| 740 | ------- |
||
| 741 | str |
||
| 742 | """ |
||
| 743 | |||
| 744 | with qdb.sql_connection.TRN: |
||
| 745 | sql = """SELECT allocation FROM |
||
| 746 | qiita.processing_job_resource_allocation |
||
| 747 | WHERE name = %s and |
||
| 748 | job_type = 'RESOURCE_PARAMS_COMMAND'""" |
||
| 749 | qdb.sql_connection.TRN.add(sql, [self.name]) |
||
| 750 | |||
| 751 | result = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 752 | |||
| 753 | # if no matches for both type and name were found, query the |
||
| 754 | # 'default' value for the type |
||
| 755 | |||
| 756 | if not result: |
||
| 757 | sql = """SELECT allocation FROM |
||
| 758 | qiita.processing_job_resource_allocation WHERE |
||
| 759 | name = %s and job_type = 'RESOURCE_PARAMS_COMMAND'""" |
||
| 760 | qdb.sql_connection.TRN.add(sql, ['default']) |
||
| 761 | |||
| 762 | result = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 763 | if not result: |
||
| 764 | raise ValueError("Could not match '%s' to a resource " |
||
| 765 | "allocation!" % self.name) |
||
| 766 | |||
| 767 | return result[0] |
||
| 768 | |||
| 769 | @property |
||
| 770 | def processing_jobs(self): |
||
| 771 | """All the processing_jobs that used this command |
||
| 772 | |||
| 773 | Returns |
||
| 774 | ------- |
||
| 775 | list of qiita_db.processing_job.ProcessingJob |
||
| 776 | List of jobs that used this command. |
||
| 777 | """ |
||
| 778 | |||
| 779 | with qdb.sql_connection.TRN: |
||
| 780 | sql = """SELECT processing_job_id FROM |
||
| 781 | qiita.processing_job |
||
| 782 | WHERE command_id = %s""" |
||
| 783 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 784 | |||
| 785 | jids = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 786 | |||
| 787 | return [qdb.processing_job.ProcessingJob(j) for j in jids] |
||
| 788 | |||
| 789 | |||
| 790 | class Software(qdb.base.QiitaObject): |
||
| 791 | r"""A software package available in the system |
||
| 792 | |||
| 793 | Attributes |
||
| 794 | ---------- |
||
| 795 | name |
||
| 796 | version |
||
| 797 | description |
||
| 798 | commands |
||
| 799 | publications |
||
| 800 | environment_name |
||
| 801 | start_script |
||
| 802 | |||
| 803 | Methods |
||
| 804 | ------- |
||
| 805 | add_publications |
||
| 806 | create |
||
| 807 | |||
| 808 | See Also |
||
| 809 | -------- |
||
| 810 | qiita_db.software.Command |
||
| 811 | """ |
||
| 812 | _table = "software" |
||
| 813 | |||
| 814 | @classmethod |
||
| 815 | def iter(cls, active=True): |
||
| 816 | """Iterates over all active software |
||
| 817 | |||
| 818 | Parameters |
||
| 819 | ---------- |
||
| 820 | active : bool, optional |
||
| 821 | If True will only return active software |
||
| 822 | |||
| 823 | Returns |
||
| 824 | ------- |
||
| 825 | list of qiita_db.software.Software |
||
| 826 | The software objects |
||
| 827 | """ |
||
| 828 | sql = """SELECT software_id |
||
| 829 | FROM qiita.software {0} |
||
| 830 | ORDER BY software_id""".format( |
||
| 831 | 'WHERE active = True' if active else '') |
||
| 832 | with qdb.sql_connection.TRN: |
||
| 833 | qdb.sql_connection.TRN.add(sql) |
||
| 834 | for s_id in qdb.sql_connection.TRN.execute_fetchflatten(): |
||
| 835 | yield cls(s_id) |
||
| 836 | |||
| 837 | @classmethod |
||
| 838 | def deactivate_all(cls): |
||
| 839 | """Deactivates all the plugins in the system""" |
||
| 840 | with qdb.sql_connection.TRN: |
||
| 841 | sql = "UPDATE qiita.software SET active = False" |
||
| 842 | qdb.sql_connection.TRN.add(sql) |
||
| 843 | sql = "UPDATE qiita.software_command SET active = False" |
||
| 844 | qdb.sql_connection.TRN.add(sql) |
||
| 845 | qdb.sql_connection.TRN.execute() |
||
| 846 | |||
| 847 | @classmethod |
||
| 848 | def from_file(cls, fp, update=False): |
||
| 849 | """Installs/updates a plugin from a plugin configuration file |
||
| 850 | |||
| 851 | Parameters |
||
| 852 | ---------- |
||
| 853 | fp : str |
||
| 854 | Path to the plugin configuration file |
||
| 855 | update : bool, optional |
||
| 856 | If true, update the values in the database with the current values |
||
| 857 | in the config file. Otherwise, use stored values and warn if config |
||
| 858 | file contents and database contents do not match |
||
| 859 | |||
| 860 | Returns |
||
| 861 | ------- |
||
| 862 | qiita_db.software.Software |
||
| 863 | The software object for the contents of `fp` |
||
| 864 | |||
| 865 | Raises |
||
| 866 | ------ |
||
| 867 | qiita_db.exceptions.QiitaDBOperationNotPermittedError |
||
| 868 | If the plugin type in the DB and in the config file doesn't match |
||
| 869 | If the (client_id, client_secret) pair in the DB and in the config |
||
| 870 | file doesn't match |
||
| 871 | """ |
||
| 872 | config = ConfigParser() |
||
| 873 | with open(fp, newline=None) as conf_file: |
||
| 874 | config.read_file(conf_file) |
||
| 875 | |||
| 876 | name = config.get('main', 'NAME') |
||
| 877 | version = config.get('main', 'VERSION') |
||
| 878 | description = config.get('main', 'DESCRIPTION') |
||
| 879 | env_script = config.get('main', 'ENVIRONMENT_SCRIPT') |
||
| 880 | start_script = config.get('main', 'START_SCRIPT') |
||
| 881 | software_type = config.get('main', 'PLUGIN_TYPE') |
||
| 882 | publications = config.get('main', 'PUBLICATIONS') |
||
| 883 | publications = loads(publications) if publications else [] |
||
| 884 | client_id = config.get('oauth2', 'CLIENT_ID') |
||
| 885 | client_secret = config.get('oauth2', 'CLIENT_SECRET') |
||
| 886 | |||
| 887 | if cls.exists(name, version): |
||
| 888 | # This plugin already exists, check that all the values are the |
||
| 889 | # same and return the existing plugin |
||
| 890 | with qdb.sql_connection.TRN: |
||
| 891 | sql = """SELECT software_id |
||
| 892 | FROM qiita.software |
||
| 893 | WHERE name = %s AND version = %s""" |
||
| 894 | qdb.sql_connection.TRN.add(sql, [name, version]) |
||
| 895 | instance = cls(qdb.sql_connection.TRN.execute_fetchlast()) |
||
| 896 | |||
| 897 | warning_values = [] |
||
| 898 | sql_update = """UPDATE qiita.software |
||
| 899 | SET {0} = %s |
||
| 900 | WHERE software_id = %s""" |
||
| 901 | |||
| 902 | values = [description, env_script, start_script] |
||
| 903 | attrs = ['description', 'environment_script', 'start_script'] |
||
| 904 | for value, attr in zip(values, attrs): |
||
| 905 | if value != instance.__getattribute__(attr): |
||
| 906 | if update: |
||
| 907 | qdb.sql_connection.TRN.add( |
||
| 908 | sql_update.format(attr), [value, instance.id]) |
||
| 909 | else: |
||
| 910 | warning_values.append(attr) |
||
| 911 | |||
| 912 | # Having a different plugin type should be an error, |
||
| 913 | # independently if the user is trying to update plugins or not |
||
| 914 | if software_type != instance.type: |
||
| 915 | raise qdb.exceptions.QiitaDBOperationNotPermittedError( |
||
| 916 | 'The plugin type of the plugin "%s" version %s does ' |
||
| 917 | 'not match the one in the system' % (name, version)) |
||
| 918 | |||
| 919 | if publications != instance.publications: |
||
| 920 | if update: |
||
| 921 | instance.add_publications(publications) |
||
| 922 | else: |
||
| 923 | warning_values.append('publications') |
||
| 924 | |||
| 925 | if (client_id != instance.client_id or |
||
| 926 | client_secret != instance.client_secret): |
||
| 927 | if update: |
||
| 928 | sql = """INSERT INTO qiita.oauth_identifiers |
||
| 929 | (client_id, client_secret) |
||
| 930 | SELECT %s, %s |
||
| 931 | WHERE NOT EXISTS(SELECT * |
||
| 932 | FROM qiita.oauth_identifiers |
||
| 933 | WHERE client_id = %s |
||
| 934 | AND client_secret = %s)""" |
||
| 935 | qdb.sql_connection.TRN.add( |
||
| 936 | sql, [client_id, client_secret, |
||
| 937 | client_id, client_secret]) |
||
| 938 | sql = """UPDATE qiita.oauth_software |
||
| 939 | SET client_id = %s |
||
| 940 | WHERE software_id = %s""" |
||
| 941 | qdb.sql_connection.TRN.add( |
||
| 942 | sql, [client_id, instance.id]) |
||
| 943 | else: |
||
| 944 | raise qdb.exceptions.QiitaDBOperationNotPermittedError( |
||
| 945 | 'The (client_id, client_secret) pair of the ' |
||
| 946 | 'plugin "%s" version "%s" does not match the one ' |
||
| 947 | 'in the system' % (name, version)) |
||
| 948 | |||
| 949 | if warning_values: |
||
| 950 | warnings.warn( |
||
| 951 | 'Plugin "%s" version "%s" config file does not match ' |
||
| 952 | 'with stored information. Check the config file or ' |
||
| 953 | 'run "qiita plugin update" to update the plugin ' |
||
| 954 | 'information. Offending values: %s' |
||
| 955 | % (name, version, ", ".join(sorted(warning_values))), |
||
| 956 | qdb.exceptions.QiitaDBWarning) |
||
| 957 | qdb.sql_connection.TRN.execute() |
||
| 958 | else: |
||
| 959 | # This is a new plugin, create it |
||
| 960 | instance = cls.create( |
||
| 961 | name, version, description, env_script, start_script, |
||
| 962 | software_type, publications=publications, client_id=client_id, |
||
| 963 | client_secret=client_secret) |
||
| 964 | |||
| 965 | return instance |
||
| 966 | |||
| 967 | @classmethod |
||
| 968 | def exists(cls, name, version): |
||
| 969 | """Returns whether the plugin (name, version) already exists |
||
| 970 | |||
| 971 | Parameters |
||
| 972 | ---------- |
||
| 973 | name : str |
||
| 974 | The name of the plugin |
||
| 975 | version : str |
||
| 976 | The version of the plugin |
||
| 977 | """ |
||
| 978 | with qdb.sql_connection.TRN: |
||
| 979 | sql = """SELECT EXISTS( |
||
| 980 | SELECT * FROM qiita.software |
||
| 981 | WHERE name = %s AND version = %s)""" |
||
| 982 | qdb.sql_connection.TRN.add(sql, [name, version]) |
||
| 983 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 984 | |||
| 985 | @classmethod |
||
| 986 | def create(cls, name, version, description, environment_script, |
||
| 987 | start_script, software_type, publications=None, |
||
| 988 | client_id=None, client_secret=None): |
||
| 989 | r"""Creates a new software in the system |
||
| 990 | |||
| 991 | Parameters |
||
| 992 | ---------- |
||
| 993 | name : str |
||
| 994 | The name of the software |
||
| 995 | version : str |
||
| 996 | The version of the software |
||
| 997 | description : str |
||
| 998 | The description of the software |
||
| 999 | environment_script : str |
||
| 1000 | The script used to start the environment in which the plugin runs |
||
| 1001 | start_script : str |
||
| 1002 | The script used to start the plugin |
||
| 1003 | software_type : str |
||
| 1004 | The type of the software |
||
| 1005 | publications : list of (str, str), optional |
||
| 1006 | A list with the (DOI, pubmed_id) of the publications attached to |
||
| 1007 | the software |
||
| 1008 | client_id : str, optional |
||
| 1009 | The client_id of the software. Default: randomly generated |
||
| 1010 | client_secret : str, optional |
||
| 1011 | The client_secret of the software. Default: randomly generated |
||
| 1012 | |||
| 1013 | Raises |
||
| 1014 | ------ |
||
| 1015 | qiita_db.exceptions.QiitaDBError |
||
| 1016 | If one of client_id or client_secret is provided but not both |
||
| 1017 | """ |
||
| 1018 | with qdb.sql_connection.TRN: |
||
| 1019 | sql = """INSERT INTO qiita.software |
||
| 1020 | (name, version, description, environment_script, |
||
| 1021 | start_script, software_type_id) |
||
| 1022 | VALUES (%s, %s, %s, %s, %s, %s) |
||
| 1023 | RETURNING software_id""" |
||
| 1024 | type_id = qdb.util.convert_to_id(software_type, "software_type") |
||
| 1025 | sql_params = [name, version, description, environment_script, |
||
| 1026 | start_script, type_id] |
||
| 1027 | qdb.sql_connection.TRN.add(sql, sql_params) |
||
| 1028 | s_id = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1029 | |||
| 1030 | instance = cls(s_id) |
||
| 1031 | |||
| 1032 | if publications: |
||
| 1033 | instance.add_publications(publications) |
||
| 1034 | |||
| 1035 | id_is_none = client_id is None |
||
| 1036 | secret_is_none = client_secret is None |
||
| 1037 | |||
| 1038 | if id_is_none and secret_is_none: |
||
| 1039 | # Both are none, generate new ones |
||
| 1040 | client_id = qdb.util.create_rand_string(50, punct=False) |
||
| 1041 | client_secret = qdb.util.create_rand_string(255, punct=False) |
||
| 1042 | elif id_is_none ^ secret_is_none: |
||
| 1043 | # One has been provided but not the other, raise an error |
||
| 1044 | raise qdb.exceptions.QiitaDBError( |
||
| 1045 | 'Plugin "%s" version "%s" cannot be created, please ' |
||
| 1046 | 'provide both client_id and client_secret or none of them' |
||
| 1047 | % (name, version)) |
||
| 1048 | |||
| 1049 | # At this point both client_id and client_secret are defined |
||
| 1050 | sql = """INSERT INTO qiita.oauth_identifiers |
||
| 1051 | (client_id, client_secret) |
||
| 1052 | SELECT %s, %s |
||
| 1053 | WHERE NOT EXISTS(SELECT * |
||
| 1054 | FROM qiita.oauth_identifiers |
||
| 1055 | WHERE client_id = %s |
||
| 1056 | AND client_secret = %s)""" |
||
| 1057 | qdb.sql_connection.TRN.add( |
||
| 1058 | sql, [client_id, client_secret, client_id, client_secret]) |
||
| 1059 | sql = """INSERT INTO qiita.oauth_software (software_id, client_id) |
||
| 1060 | VALUES (%s, %s)""" |
||
| 1061 | qdb.sql_connection.TRN.add(sql, [s_id, client_id]) |
||
| 1062 | |||
| 1063 | return instance |
||
| 1064 | |||
| 1065 | @classmethod |
||
| 1066 | def from_name_and_version(cls, name, version): |
||
| 1067 | """Returns the software object with the given name and version |
||
| 1068 | |||
| 1069 | Parameters |
||
| 1070 | ---------- |
||
| 1071 | name: str |
||
| 1072 | The software name |
||
| 1073 | version : str |
||
| 1074 | The software version |
||
| 1075 | |||
| 1076 | Returns |
||
| 1077 | ------- |
||
| 1078 | qiita_db.software.Software |
||
| 1079 | The software with the given name and version |
||
| 1080 | |||
| 1081 | Raises |
||
| 1082 | ------ |
||
| 1083 | qiita_db.exceptions.QiitaDBUnknownIDError |
||
| 1084 | If no software with the given name and version exists |
||
| 1085 | """ |
||
| 1086 | with qdb.sql_connection.TRN: |
||
| 1087 | sql = """SELECT software_id |
||
| 1088 | FROM qiita.software |
||
| 1089 | WHERE name = %s AND version = %s""" |
||
| 1090 | qdb.sql_connection.TRN.add(sql, [name, version]) |
||
| 1091 | res = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 1092 | if not res: |
||
| 1093 | raise qdb.exceptions.QiitaDBUnknownIDError( |
||
| 1094 | "%s %s" % (name, version), cls._table) |
||
| 1095 | return cls(res[0][0]) |
||
| 1096 | |||
| 1097 | @property |
||
| 1098 | def name(self): |
||
| 1099 | """The name of the software |
||
| 1100 | |||
| 1101 | Returns |
||
| 1102 | ------- |
||
| 1103 | str |
||
| 1104 | The name of the software |
||
| 1105 | """ |
||
| 1106 | with qdb.sql_connection.TRN: |
||
| 1107 | sql = "SELECT name FROM qiita.software WHERE software_id = %s" |
||
| 1108 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1109 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1110 | |||
| 1111 | @property |
||
| 1112 | def version(self): |
||
| 1113 | """The version of the software |
||
| 1114 | |||
| 1115 | Returns |
||
| 1116 | ------- |
||
| 1117 | str |
||
| 1118 | The version of the software |
||
| 1119 | """ |
||
| 1120 | with qdb.sql_connection.TRN: |
||
| 1121 | sql = "SELECT version FROM qiita.software WHERE software_id = %s" |
||
| 1122 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1123 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1124 | |||
| 1125 | @property |
||
| 1126 | def description(self): |
||
| 1127 | """The description of the software |
||
| 1128 | |||
| 1129 | Returns |
||
| 1130 | ------- |
||
| 1131 | str |
||
| 1132 | The software description |
||
| 1133 | """ |
||
| 1134 | with qdb.sql_connection.TRN: |
||
| 1135 | sql = """SELECT description |
||
| 1136 | FROM qiita.software |
||
| 1137 | WHERE software_id = %s""" |
||
| 1138 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1139 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1140 | |||
| 1141 | @property |
||
| 1142 | def commands(self): |
||
| 1143 | """The list of commands attached to this software |
||
| 1144 | |||
| 1145 | Returns |
||
| 1146 | ------- |
||
| 1147 | list of qiita_db.software.Command |
||
| 1148 | The commands attached to this software package |
||
| 1149 | """ |
||
| 1150 | with qdb.sql_connection.TRN: |
||
| 1151 | sql = """SELECT command_id |
||
| 1152 | FROM qiita.software_command |
||
| 1153 | WHERE software_id = %s""" |
||
| 1154 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1155 | return [Command(cid) |
||
| 1156 | for cid in qdb.sql_connection.TRN.execute_fetchflatten()] |
||
| 1157 | |||
| 1158 | def get_command(self, cmd_name): |
||
| 1159 | """Returns the command with the given name in the software |
||
| 1160 | |||
| 1161 | Parameters |
||
| 1162 | ---------- |
||
| 1163 | cmd_name: str |
||
| 1164 | The command with the given name |
||
| 1165 | |||
| 1166 | Returns |
||
| 1167 | ------- |
||
| 1168 | qiita_db.software.Command |
||
| 1169 | The command with the given name in this software |
||
| 1170 | """ |
||
| 1171 | with qdb.sql_connection.TRN: |
||
| 1172 | sql = """SELECT command_id |
||
| 1173 | FROM qiita.software_command |
||
| 1174 | WHERE software_id =%s AND name=%s""" |
||
| 1175 | qdb.sql_connection.TRN.add(sql, [self.id, cmd_name]) |
||
| 1176 | res = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 1177 | if not res: |
||
| 1178 | raise qdb.exceptions.QiitaDBUnknownIDError( |
||
| 1179 | cmd_name, "software_command") |
||
| 1180 | return Command(res[0][0]) |
||
| 1181 | |||
| 1182 | @property |
||
| 1183 | def publications(self): |
||
| 1184 | """The publications attached to the software |
||
| 1185 | |||
| 1186 | Returns |
||
| 1187 | ------- |
||
| 1188 | list of (str, str) |
||
| 1189 | The list of DOI and pubmed_id attached to the publication |
||
| 1190 | """ |
||
| 1191 | with qdb.sql_connection.TRN: |
||
| 1192 | sql = """SELECT p.doi, p.pubmed_id |
||
| 1193 | FROM qiita.publication p |
||
| 1194 | JOIN qiita.software_publication sp |
||
| 1195 | ON p.doi = sp.publication_doi |
||
| 1196 | WHERE sp.software_id = %s""" |
||
| 1197 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1198 | return qdb.sql_connection.TRN.execute_fetchindex() |
||
| 1199 | |||
| 1200 | def add_publications(self, publications): |
||
| 1201 | """Add publications to the software |
||
| 1202 | |||
| 1203 | Parameters |
||
| 1204 | ---------- |
||
| 1205 | publications : list of 2-tuples of str |
||
| 1206 | A list with the (DOI, pubmed_id) of the publications to be attached |
||
| 1207 | to the software |
||
| 1208 | |||
| 1209 | Notes |
||
| 1210 | ----- |
||
| 1211 | For more information about pubmed id, visit |
||
| 1212 | https://www.nlm.nih.gov/bsd/disted/pubmedtutorial/020_830.html |
||
| 1213 | """ |
||
| 1214 | with qdb.sql_connection.TRN: |
||
| 1215 | sql = """INSERT INTO qiita.publication (doi, pubmed_id) |
||
| 1216 | SELECT %s, %s |
||
| 1217 | WHERE NOT EXISTS(SELECT * |
||
| 1218 | FROM qiita.publication |
||
| 1219 | WHERE doi = %s)""" |
||
| 1220 | args = [[doi, pid, doi] for doi, pid in publications] |
||
| 1221 | qdb.sql_connection.TRN.add(sql, args, many=True) |
||
| 1222 | |||
| 1223 | sql = """INSERT INTO qiita.software_publication |
||
| 1224 | (software_id, publication_doi) |
||
| 1225 | SELECT %s, %s |
||
| 1226 | WHERE NOT EXISTS(SELECT * |
||
| 1227 | FROM qiita.software_publication |
||
| 1228 | WHERE software_id = %s AND |
||
| 1229 | publication_doi = %s)""" |
||
| 1230 | sql_params = [[self.id, doi, self.id, doi] |
||
| 1231 | for doi, _ in publications] |
||
| 1232 | qdb.sql_connection.TRN.add(sql, sql_params, many=True) |
||
| 1233 | qdb.sql_connection.TRN.execute() |
||
| 1234 | |||
| 1235 | @property |
||
| 1236 | def environment_script(self): |
||
| 1237 | """The script used to start the plugin environment |
||
| 1238 | |||
| 1239 | Returns |
||
| 1240 | ------- |
||
| 1241 | str |
||
| 1242 | The script used to start the environment |
||
| 1243 | """ |
||
| 1244 | with qdb.sql_connection.TRN: |
||
| 1245 | sql = """SELECT environment_script |
||
| 1246 | FROM qiita.software |
||
| 1247 | WHERE software_id = %s""" |
||
| 1248 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1249 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1250 | |||
| 1251 | @property |
||
| 1252 | def start_script(self): |
||
| 1253 | """The script used to start the plugin |
||
| 1254 | |||
| 1255 | Returns |
||
| 1256 | ------- |
||
| 1257 | str |
||
| 1258 | The plugin's start script |
||
| 1259 | """ |
||
| 1260 | with qdb.sql_connection.TRN: |
||
| 1261 | sql = """SELECT start_script |
||
| 1262 | FROM qiita.software |
||
| 1263 | WHERE software_id = %s""" |
||
| 1264 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1265 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1266 | |||
| 1267 | @property |
||
| 1268 | def type(self): |
||
| 1269 | """Returns the type of the software |
||
| 1270 | |||
| 1271 | Returns |
||
| 1272 | ------- |
||
| 1273 | str |
||
| 1274 | The type of the software |
||
| 1275 | """ |
||
| 1276 | with qdb.sql_connection.TRN: |
||
| 1277 | sql = """SELECT software_type |
||
| 1278 | FROM qiita.software_type |
||
| 1279 | JOIN qiita.software USING (software_type_id) |
||
| 1280 | WHERE software_id = %s""" |
||
| 1281 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1282 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1283 | |||
| 1284 | @property |
||
| 1285 | def deprecated(self): |
||
| 1286 | """Returns if the software is deprecated or not |
||
| 1287 | |||
| 1288 | Returns |
||
| 1289 | ------- |
||
| 1290 | bool |
||
| 1291 | Whether the software is deprecated or not |
||
| 1292 | """ |
||
| 1293 | with qdb.sql_connection.TRN: |
||
| 1294 | sql = """SELECT deprecated |
||
| 1295 | FROM qiita.software |
||
| 1296 | WHERE software_id = %s""" |
||
| 1297 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1298 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1299 | |||
| 1300 | @deprecated.setter |
||
| 1301 | def deprecated(self, deprecate): |
||
| 1302 | """Changes deprecated of the software |
||
| 1303 | |||
| 1304 | Parameters |
||
| 1305 | ---------- |
||
| 1306 | deprecate : bool |
||
| 1307 | New software deprecate value |
||
| 1308 | """ |
||
| 1309 | sql = """UPDATE qiita.software SET deprecated = %s |
||
| 1310 | WHERE software_id = %s""" |
||
| 1311 | qdb.sql_connection.perform_as_transaction(sql, [deprecate, self._id]) |
||
| 1312 | |||
| 1313 | @property |
||
| 1314 | def active(self): |
||
| 1315 | """Returns if the software is active or not |
||
| 1316 | |||
| 1317 | Returns |
||
| 1318 | ------- |
||
| 1319 | bool |
||
| 1320 | Whether the software is active or not |
||
| 1321 | """ |
||
| 1322 | with qdb.sql_connection.TRN: |
||
| 1323 | sql = "SELECT active FROM qiita.software WHERE software_id = %s" |
||
| 1324 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1325 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1326 | |||
| 1327 | def activate(self): |
||
| 1328 | """Activates the plugin""" |
||
| 1329 | sql = """UPDATE qiita.software |
||
| 1330 | SET active = %s |
||
| 1331 | WHERE software_id = %s""" |
||
| 1332 | qdb.sql_connection.perform_as_transaction(sql, [True, self.id]) |
||
| 1333 | |||
| 1334 | @property |
||
| 1335 | def client_id(self): |
||
| 1336 | """Returns the client id of the plugin |
||
| 1337 | |||
| 1338 | Returns |
||
| 1339 | ------- |
||
| 1340 | str |
||
| 1341 | The client id of the software |
||
| 1342 | """ |
||
| 1343 | with qdb.sql_connection.TRN: |
||
| 1344 | sql = """SELECT client_id |
||
| 1345 | FROM qiita.oauth_software |
||
| 1346 | WHERE software_id = %s""" |
||
| 1347 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1348 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1349 | |||
| 1350 | @property |
||
| 1351 | def client_secret(self): |
||
| 1352 | """Returns the client secret of the plugin |
||
| 1353 | |||
| 1354 | Returns |
||
| 1355 | ------- |
||
| 1356 | str |
||
| 1357 | The client secrect of the plugin |
||
| 1358 | """ |
||
| 1359 | with qdb.sql_connection.TRN: |
||
| 1360 | sql = """SELECT client_secret |
||
| 1361 | FROM qiita.oauth_software |
||
| 1362 | JOIN qiita.oauth_identifiers USING (client_id) |
||
| 1363 | WHERE software_id = %s""" |
||
| 1364 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1365 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1366 | |||
| 1367 | def register_commands(self): |
||
| 1368 | """Registers the software commands""" |
||
| 1369 | url = "%s%s" % (qiita_config.base_url, qiita_config.portal_dir) |
||
| 1370 | cmd = '%s; %s "%s" "register" "ignored"' % ( |
||
| 1371 | self.environment_script, self.start_script, url) |
||
| 1372 | |||
| 1373 | # it can be assumed that any command beginning with 'source' |
||
| 1374 | # is calling 'source', an internal command of 'bash' and hence |
||
| 1375 | # should be executed from bash, instead of sh. |
||
| 1376 | # TODO: confirm that exit_code propagates from bash to sh to |
||
| 1377 | # rv. |
||
| 1378 | if cmd.startswith('source'): |
||
| 1379 | cmd = "bash -c '%s'" % cmd |
||
| 1380 | |||
| 1381 | p_out, p_err, rv = qdb.processing_job._system_call(cmd) |
||
| 1382 | |||
| 1383 | if rv != 0: |
||
| 1384 | s = "cmd: %s\nexit status: %d\n" % (cmd, rv) |
||
| 1385 | s += "stdout: %s\nstderr: %s\n" % (p_out, p_err) |
||
| 1386 | |||
| 1387 | raise ValueError(s) |
||
| 1388 | |||
| 1389 | |||
| 1390 | class DefaultParameters(qdb.base.QiitaObject): |
||
| 1391 | """Models a default set of parameters of a command |
||
| 1392 | |||
| 1393 | Attributes |
||
| 1394 | ---------- |
||
| 1395 | name |
||
| 1396 | values |
||
| 1397 | |||
| 1398 | Methods |
||
| 1399 | ------- |
||
| 1400 | exists |
||
| 1401 | create |
||
| 1402 | iter |
||
| 1403 | to_str |
||
| 1404 | to_file |
||
| 1405 | |||
| 1406 | See Also |
||
| 1407 | -------- |
||
| 1408 | qiita_db.software.Command |
||
| 1409 | """ |
||
| 1410 | _table = 'default_parameter_set' |
||
| 1411 | |||
| 1412 | @classmethod |
||
| 1413 | def exists(cls, command, **kwargs): |
||
| 1414 | r"""Check if a parameter set already exists |
||
| 1415 | |||
| 1416 | Parameters |
||
| 1417 | ---------- |
||
| 1418 | command : qiita_db.software.Command |
||
| 1419 | The command to which the parameter set belongs to |
||
| 1420 | kwargs : dict of {str: str} |
||
| 1421 | The parameters and their values |
||
| 1422 | |||
| 1423 | Returns |
||
| 1424 | ------- |
||
| 1425 | bool |
||
| 1426 | Whether if the parameter set exists in the given command |
||
| 1427 | |||
| 1428 | Raises |
||
| 1429 | ------ |
||
| 1430 | qiita_db.exceptions.QiitaDBError |
||
| 1431 | - If there are missing parameters for the given command |
||
| 1432 | - If `kwargs` contains parameters not originally defined in the |
||
| 1433 | command |
||
| 1434 | """ |
||
| 1435 | with qdb.sql_connection.TRN: |
||
| 1436 | command_params = set(command.optional_parameters) |
||
| 1437 | user_params = set(kwargs) |
||
| 1438 | |||
| 1439 | missing_in_user = command_params - user_params |
||
| 1440 | extra_in_user = user_params - command_params |
||
| 1441 | |||
| 1442 | if missing_in_user or extra_in_user: |
||
| 1443 | raise qdb.exceptions.QiitaDBError( |
||
| 1444 | "The given set of parameters do not match the ones for " |
||
| 1445 | "the command.\nMissing parameters: %s\n" |
||
| 1446 | "Extra parameters: %s\n" |
||
| 1447 | % (', '.join(missing_in_user), ', '.join(extra_in_user))) |
||
| 1448 | |||
| 1449 | sql = """SELECT parameter_set |
||
| 1450 | FROM qiita.default_parameter_set |
||
| 1451 | WHERE command_id = %s""" |
||
| 1452 | qdb.sql_connection.TRN.add(sql, [command.id]) |
||
| 1453 | for p_set in qdb.sql_connection.TRN.execute_fetchflatten(): |
||
| 1454 | if p_set == kwargs: |
||
| 1455 | return True |
||
| 1456 | |||
| 1457 | return False |
||
| 1458 | |||
| 1459 | @classmethod |
||
| 1460 | def create(cls, param_set_name, command, **kwargs): |
||
| 1461 | r"""Create a new parameter set for the given command |
||
| 1462 | |||
| 1463 | Parameters |
||
| 1464 | ---------- |
||
| 1465 | param_set_name: str |
||
| 1466 | The name of the new parameter set |
||
| 1467 | command : qiita_db.software.Command |
||
| 1468 | The command to add the new parameter set |
||
| 1469 | kwargs : dict |
||
| 1470 | The parameters and their values |
||
| 1471 | |||
| 1472 | Returns |
||
| 1473 | ------- |
||
| 1474 | qiita_db.software.Parameters |
||
| 1475 | The new parameter set instance |
||
| 1476 | |||
| 1477 | Raises |
||
| 1478 | ------ |
||
| 1479 | qiita_db.exceptions.QiitaDBError |
||
| 1480 | - If there are missing parameters for the given command |
||
| 1481 | - If there are extra parameters in `kwargs` than for the given |
||
| 1482 | command |
||
| 1483 | qdb.exceptions.QiitaDBDuplicateError |
||
| 1484 | - If the parameter set already exists |
||
| 1485 | """ |
||
| 1486 | with qdb.sql_connection.TRN: |
||
| 1487 | # setting to default values all parameters not in the user_params |
||
| 1488 | cmd_params = command.optional_parameters |
||
| 1489 | missing_in_user = {k: cmd_params[k][1] |
||
| 1490 | for k in (set(cmd_params) - set(kwargs))} |
||
| 1491 | if missing_in_user: |
||
| 1492 | kwargs.update(missing_in_user) |
||
| 1493 | |||
| 1494 | # If the columns in kwargs and command do not match, cls.exists |
||
| 1495 | # will raise the error for us |
||
| 1496 | if cls.exists(command, **kwargs): |
||
| 1497 | raise qdb.exceptions.QiitaDBDuplicateError( |
||
| 1498 | cls._table, "Values: %s" % kwargs) |
||
| 1499 | |||
| 1500 | sql = """INSERT INTO qiita.default_parameter_set |
||
| 1501 | (command_id, parameter_set_name, parameter_set) |
||
| 1502 | VALUES (%s, %s, %s) |
||
| 1503 | RETURNING default_parameter_set_id""" |
||
| 1504 | sql_args = [command.id, param_set_name, dumps(kwargs)] |
||
| 1505 | qdb.sql_connection.TRN.add(sql, sql_args) |
||
| 1506 | |||
| 1507 | return cls(qdb.sql_connection.TRN.execute_fetchlast()) |
||
| 1508 | |||
| 1509 | @property |
||
| 1510 | def name(self): |
||
| 1511 | """The name of the parameter set |
||
| 1512 | |||
| 1513 | Returns |
||
| 1514 | ------- |
||
| 1515 | str |
||
| 1516 | The name of the parameter set |
||
| 1517 | """ |
||
| 1518 | with qdb.sql_connection.TRN: |
||
| 1519 | sql = """SELECT parameter_set_name |
||
| 1520 | FROM qiita.default_parameter_set |
||
| 1521 | WHERE default_parameter_set_id = %s""" |
||
| 1522 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1523 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1524 | |||
| 1525 | @property |
||
| 1526 | def values(self): |
||
| 1527 | """The values of the parameter set |
||
| 1528 | |||
| 1529 | Returns |
||
| 1530 | ------- |
||
| 1531 | dict of {str: object} |
||
| 1532 | Dictionary with the parameters values keyed by parameter name |
||
| 1533 | """ |
||
| 1534 | with qdb.sql_connection.TRN: |
||
| 1535 | sql = """SELECT parameter_set |
||
| 1536 | FROM qiita.default_parameter_set |
||
| 1537 | WHERE default_parameter_set_id = %s""" |
||
| 1538 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1539 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1540 | |||
| 1541 | @property |
||
| 1542 | def command(self): |
||
| 1543 | """The command that this parameter set belongs to |
||
| 1544 | |||
| 1545 | Returns |
||
| 1546 | ------- |
||
| 1547 | qiita_db.software.Command |
||
| 1548 | The command that this parameter set belongs to |
||
| 1549 | """ |
||
| 1550 | with qdb.sql_connection.TRN: |
||
| 1551 | sql = """SELECT command_id |
||
| 1552 | FROM qiita.default_parameter_set |
||
| 1553 | WHERE default_parameter_set_id = %s""" |
||
| 1554 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1555 | return Command(qdb.sql_connection.TRN.execute_fetchlast()) |
||
| 1556 | |||
| 1557 | |||
| 1558 | class Parameters(object): |
||
| 1559 | """Represents an instance of parameters used to process an artifact |
||
| 1560 | |||
| 1561 | Raises |
||
| 1562 | ------ |
||
| 1563 | qiita_db.exceptions.QiitaDBOperationNotPermittedError |
||
| 1564 | If trying to instantiate this class directly. In order to instantiate |
||
| 1565 | this class, the classmethods `load` or `from_default_params` should |
||
| 1566 | be used. |
||
| 1567 | """ |
||
| 1568 | |||
| 1569 | def __eq__(self, other): |
||
| 1570 | """Equality based on the parameter values and the command""" |
||
| 1571 | if type(self) is not type(other): |
||
| 1572 | return False |
||
| 1573 | if self.command != other.command: |
||
| 1574 | return False |
||
| 1575 | if self.values != other.values: |
||
| 1576 | return False |
||
| 1577 | return True |
||
| 1578 | |||
| 1579 | @classmethod |
||
| 1580 | def load(cls, command, json_str=None, values_dict=None): |
||
| 1581 | """Load the parameters set form a json str or from a dict of values |
||
| 1582 | |||
| 1583 | Parameters |
||
| 1584 | ---------- |
||
| 1585 | command : qiita_db.software.Command |
||
| 1586 | The command to which the parameter set belongs to |
||
| 1587 | json_str : str, optional |
||
| 1588 | The json string encoding the parameters |
||
| 1589 | values_dict : dict of {str: object}, optional |
||
| 1590 | The dictionary with the parameter values |
||
| 1591 | |||
| 1592 | Returns |
||
| 1593 | ------- |
||
| 1594 | qiita_db.software.Parameters |
||
| 1595 | The loaded parameter set |
||
| 1596 | |||
| 1597 | Raises |
||
| 1598 | ------ |
||
| 1599 | qiita_db.exceptions.QiitaDBError |
||
| 1600 | - If `json_str` and `values` are both provided |
||
| 1601 | - If neither `json_str` or `values` are provided |
||
| 1602 | - If `json_str` or `values` do not encode a parameter set of |
||
| 1603 | the provided command. |
||
| 1604 | |||
| 1605 | Notes |
||
| 1606 | ----- |
||
| 1607 | The parameters `json_str` and `values_dict` are mutually exclusive, |
||
| 1608 | only one of them should be provided at a time. However, one of them |
||
| 1609 | should always be provided. |
||
| 1610 | """ |
||
| 1611 | if json_str is None and values_dict is None: |
||
| 1612 | raise qdb.exceptions.QiitaDBError( |
||
| 1613 | "Either `json_str` or `values_dict` should be provided.") |
||
| 1614 | elif json_str is not None and values_dict is not None: |
||
| 1615 | raise qdb.exceptions.QiitaDBError( |
||
| 1616 | "Either `json_str` or `values_dict` should be provided, " |
||
| 1617 | "but not both") |
||
| 1618 | elif json_str is not None: |
||
| 1619 | parameters = loads(json_str) |
||
| 1620 | error_msg = ("The provided JSON string doesn't encode a " |
||
| 1621 | "parameter set for command %s" % command.id) |
||
| 1622 | else: |
||
| 1623 | if not isinstance(values_dict, dict): |
||
| 1624 | raise qdb.exceptions.QiitaDBError( |
||
| 1625 | "The provided value_dict is %s (i.e. not None) but also " |
||
| 1626 | "not a dictionary for command %s" % ( |
||
| 1627 | values_dict, command.id)) |
||
| 1628 | parameters = deepcopy(values_dict) |
||
| 1629 | error_msg = ("The provided values dictionary doesn't encode a " |
||
| 1630 | "parameter set for command %s" % command.id) |
||
| 1631 | |||
| 1632 | # setting to default values all parameters not in the user_params |
||
| 1633 | cmd_params = command.optional_parameters |
||
| 1634 | missing_in_user = {k: cmd_params[k][1] |
||
| 1635 | for k in (set(cmd_params) - set(parameters))} |
||
| 1636 | if missing_in_user: |
||
| 1637 | parameters.update(missing_in_user) |
||
| 1638 | |||
| 1639 | with qdb.sql_connection.TRN: |
||
| 1640 | cmd_reqd_params = command.required_parameters |
||
| 1641 | cmd_opt_params = command.optional_parameters |
||
| 1642 | |||
| 1643 | values = {} |
||
| 1644 | for key in cmd_reqd_params: |
||
| 1645 | try: |
||
| 1646 | values[key] = parameters.pop(key) |
||
| 1647 | except KeyError: |
||
| 1648 | raise qdb.exceptions.QiitaDBError( |
||
| 1649 | "%s. Missing required parameter: %s" |
||
| 1650 | % (error_msg, key)) |
||
| 1651 | |||
| 1652 | for key in cmd_opt_params: |
||
| 1653 | try: |
||
| 1654 | values[key] = parameters.pop(key) |
||
| 1655 | except KeyError: |
||
| 1656 | raise qdb.exceptions.QiitaDBError( |
||
| 1657 | "%s. Missing optional parameter: %s" |
||
| 1658 | % (error_msg, key)) |
||
| 1659 | |||
| 1660 | if parameters: |
||
| 1661 | raise qdb.exceptions.QiitaDBError( |
||
| 1662 | "%s. Extra parameters: %s" |
||
| 1663 | % (error_msg, ', '.join(parameters.keys()))) |
||
| 1664 | |||
| 1665 | return cls(values, command) |
||
| 1666 | |||
| 1667 | @classmethod |
||
| 1668 | def from_default_params(cls, dflt_params, req_params, opt_params=None): |
||
| 1669 | """Creates the parameter set from a `dflt_params` set |
||
| 1670 | |||
| 1671 | Parameters |
||
| 1672 | ---------- |
||
| 1673 | dflt_params : qiita_db.software.DefaultParameters |
||
| 1674 | The DefaultParameters object in which this instance is based on |
||
| 1675 | req_params : dict of {str: object} |
||
| 1676 | The required parameters values, keyed by parameter name |
||
| 1677 | opt_params : dict of {str: object}, optional |
||
| 1678 | The optional parameters to change from the default set, keyed by |
||
| 1679 | parameter name. Default: None, use the values in `dflt_params` |
||
| 1680 | |||
| 1681 | Raises |
||
| 1682 | ------ |
||
| 1683 | QiitaDBError |
||
| 1684 | - If there are missing requried parameters |
||
| 1685 | - If there is an unknown required ot optional parameter |
||
| 1686 | """ |
||
| 1687 | with qdb.sql_connection.TRN: |
||
| 1688 | command = dflt_params.command |
||
| 1689 | cmd_req_params = command.required_parameters |
||
| 1690 | cmd_opt_params = command.optional_parameters |
||
| 1691 | |||
| 1692 | missing_reqd = set(cmd_req_params) - set(req_params) |
||
| 1693 | extra_reqd = set(req_params) - set(cmd_req_params) |
||
| 1694 | if missing_reqd or extra_reqd: |
||
| 1695 | raise qdb.exceptions.QiitaDBError( |
||
| 1696 | "Provided required parameters not expected.\n" |
||
| 1697 | "Missing required parameters: %s\n" |
||
| 1698 | "Extra required parameters: %s\n" |
||
| 1699 | % (', '.join(missing_reqd), ', '.join(extra_reqd))) |
||
| 1700 | |||
| 1701 | if opt_params: |
||
| 1702 | extra_opts = set(opt_params) - set(cmd_opt_params) |
||
| 1703 | if extra_opts: |
||
| 1704 | raise qdb.exceptions.QiitaDBError( |
||
| 1705 | "Extra optional parameters provded: %s" |
||
| 1706 | % ', '.join(extra_opts)) |
||
| 1707 | |||
| 1708 | values = dflt_params.values |
||
| 1709 | values.update(req_params) |
||
| 1710 | |||
| 1711 | if opt_params: |
||
| 1712 | values.update(opt_params) |
||
| 1713 | |||
| 1714 | return cls(values, command) |
||
| 1715 | |||
| 1716 | def __init__(self, values, command): |
||
| 1717 | # Time for some python magic! The __init__ function should not be used |
||
| 1718 | # outside of this module, users should always be using one of the above |
||
| 1719 | # classmethods to instantiate the object. Lets test that it is the case |
||
| 1720 | # First, we are going to get the current frame (i.e. this __init__) |
||
| 1721 | # function and the caller to the __init__ |
||
| 1722 | current_frame = inspect.currentframe() |
||
| 1723 | caller_frame = current_frame.f_back |
||
| 1724 | # The file names where the function is defined is stored in the |
||
| 1725 | # f_code.co_filename attribute, and in this case it has to be the same |
||
| 1726 | # for both of them. Also, we are restricing that the name of the caller |
||
| 1727 | # should be either `load` or `from_default_params`, which are the two |
||
| 1728 | # classmethods defined above |
||
| 1729 | current_file = current_frame.f_code.co_filename |
||
| 1730 | caller_file = caller_frame.f_code.co_filename |
||
| 1731 | caller_name = caller_frame.f_code.co_name |
||
| 1732 | if current_file != caller_file or \ |
||
| 1733 | caller_name not in ['load', 'from_default_params']: |
||
| 1734 | raise qdb.exceptions.QiitaDBOperationNotPermittedError( |
||
| 1735 | "qiita_db.software.Parameters can't be instantiated directly. " |
||
| 1736 | "Please use one of the classmethods: `load` or " |
||
| 1737 | "`from_default_params`") |
||
| 1738 | |||
| 1739 | self._values = values |
||
| 1740 | self._command = command |
||
| 1741 | |||
| 1742 | @property |
||
| 1743 | def command(self): |
||
| 1744 | """The command to which this parameter set belongs to |
||
| 1745 | |||
| 1746 | Returns |
||
| 1747 | ------- |
||
| 1748 | qiita_db.software.Command |
||
| 1749 | The command to which this parameter set belongs to |
||
| 1750 | """ |
||
| 1751 | return self._command |
||
| 1752 | |||
| 1753 | @property |
||
| 1754 | def values(self): |
||
| 1755 | """The values of the parameters |
||
| 1756 | |||
| 1757 | Returns |
||
| 1758 | ------- |
||
| 1759 | dict of {str: object} |
||
| 1760 | The parameter values keyed by parameter name |
||
| 1761 | """ |
||
| 1762 | return self._values |
||
| 1763 | |||
| 1764 | def dump(self): |
||
| 1765 | """Return the values in the parameter as JSON |
||
| 1766 | |||
| 1767 | Returns |
||
| 1768 | ------- |
||
| 1769 | str |
||
| 1770 | The parameter values as a JSON string |
||
| 1771 | """ |
||
| 1772 | return dumps(self._values, sort_keys=True) |
||
| 1773 | |||
| 1774 | |||
| 1775 | class DefaultWorkflowNode(qdb.base.QiitaObject): |
||
| 1776 | r"""Represents a node in a default software workflow |
||
| 1777 | |||
| 1778 | Attributes |
||
| 1779 | ---------- |
||
| 1780 | command |
||
| 1781 | parameters |
||
| 1782 | """ |
||
| 1783 | _table = "default_workflow_node" |
||
| 1784 | |||
| 1785 | @property |
||
| 1786 | def default_parameter(self): |
||
| 1787 | """The default parameter set to use in this node |
||
| 1788 | |||
| 1789 | Returns |
||
| 1790 | ------- |
||
| 1791 | qiita_db.software.DefaultParameters |
||
| 1792 | """ |
||
| 1793 | with qdb.sql_connection.TRN: |
||
| 1794 | sql = """SELECT default_parameter_set_id |
||
| 1795 | FROM qiita.default_workflow_node |
||
| 1796 | WHERE default_workflow_node_id = %s""" |
||
| 1797 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1798 | params_id = qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1799 | return qdb.software.DefaultParameters(params_id) |
||
| 1800 | |||
| 1801 | |||
| 1802 | class DefaultWorkflowEdge(qdb.base.QiitaObject): |
||
| 1803 | r"""Represents an edge in a default software workflow |
||
| 1804 | |||
| 1805 | Attributes |
||
| 1806 | ---------- |
||
| 1807 | connections |
||
| 1808 | """ |
||
| 1809 | _table = "default_workflow_edge" |
||
| 1810 | |||
| 1811 | @property |
||
| 1812 | def connections(self): |
||
| 1813 | """Retrieve how the commands are connected using this edge |
||
| 1814 | |||
| 1815 | Returns |
||
| 1816 | ------- |
||
| 1817 | list of [str, str] |
||
| 1818 | The list of pairs of output parameter name and input parameter name |
||
| 1819 | used to connect the output of the source command to the input of |
||
| 1820 | the destination command. |
||
| 1821 | """ |
||
| 1822 | with qdb.sql_connection.TRN: |
||
| 1823 | sql = """SELECT name, parameter_name, artifact_type |
||
| 1824 | FROM qiita.default_workflow_edge_connections c |
||
| 1825 | JOIN qiita.command_output o |
||
| 1826 | ON c.parent_output_id = o.command_output_id |
||
| 1827 | JOIN qiita.command_parameter p |
||
| 1828 | ON c.child_input_id = p.command_parameter_id |
||
| 1829 | LEFT JOIN qiita.artifact_type USING (artifact_type_id) |
||
| 1830 | WHERE default_workflow_edge_id = %s""" |
||
| 1831 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1832 | return qdb.sql_connection.TRN.execute_fetchindex() |
||
| 1833 | |||
| 1834 | |||
| 1835 | class DefaultWorkflow(qdb.base.QiitaObject): |
||
| 1836 | r"""Represents a software's default workflow |
||
| 1837 | |||
| 1838 | A default workflow is defined by a Directed Acyclic Graph (DAG) in which |
||
| 1839 | the nodes represent the commands to be executed with the default parameter |
||
| 1840 | set to use and the edges represent the command precedence, including |
||
| 1841 | which outputs of the source command are provided as input to the |
||
| 1842 | destination command. |
||
| 1843 | """ |
||
| 1844 | _table = "default_workflow" |
||
| 1845 | |||
| 1846 | @classmethod |
||
| 1847 | def iter(cls, active=True): |
||
| 1848 | """Iterates over all active DefaultWorkflow |
||
| 1849 | |||
| 1850 | Parameters |
||
| 1851 | ---------- |
||
| 1852 | active : bool, optional |
||
| 1853 | If True will only return active software |
||
| 1854 | |||
| 1855 | Returns |
||
| 1856 | ------- |
||
| 1857 | list of qiita_db.software.DefaultWorkflow |
||
| 1858 | The DefaultWorkflow objects |
||
| 1859 | """ |
||
| 1860 | sql = """SELECT default_workflow_id |
||
| 1861 | FROM qiita.default_workflow {0} |
||
| 1862 | ORDER BY default_workflow_id""".format( |
||
| 1863 | 'WHERE active = True' if active else '') |
||
| 1864 | with qdb.sql_connection.TRN: |
||
| 1865 | qdb.sql_connection.TRN.add(sql) |
||
| 1866 | for s_id in qdb.sql_connection.TRN.execute_fetchflatten(): |
||
| 1867 | yield cls(s_id) |
||
| 1868 | |||
| 1869 | @property |
||
| 1870 | def active(self): |
||
| 1871 | """Retrieves active status of the default workflow |
||
| 1872 | |||
| 1873 | Returns |
||
| 1874 | ------- |
||
| 1875 | active : bool |
||
| 1876 | active value |
||
| 1877 | """ |
||
| 1878 | with qdb.sql_connection.TRN: |
||
| 1879 | sql = """SELECT active |
||
| 1880 | FROM qiita.default_workflow |
||
| 1881 | WHERE default_workflow_id = %s""" |
||
| 1882 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1883 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1884 | |||
| 1885 | @active.setter |
||
| 1886 | def active(self, active): |
||
| 1887 | """Changes active status of the default workflow |
||
| 1888 | |||
| 1889 | Parameters |
||
| 1890 | ---------- |
||
| 1891 | active : bool |
||
| 1892 | New active value |
||
| 1893 | """ |
||
| 1894 | sql = """UPDATE qiita.default_workflow SET active = %s |
||
| 1895 | WHERE default_workflow_id = %s""" |
||
| 1896 | qdb.sql_connection.perform_as_transaction(sql, [active, self._id]) |
||
| 1897 | |||
| 1898 | @property |
||
| 1899 | def name(self): |
||
| 1900 | with qdb.sql_connection.TRN: |
||
| 1901 | sql = """SELECT name |
||
| 1902 | FROM qiita.default_workflow |
||
| 1903 | WHERE default_workflow_id = %s""" |
||
| 1904 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1905 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1906 | |||
| 1907 | @property |
||
| 1908 | def description(self): |
||
| 1909 | """Retrieves the description of the default workflow |
||
| 1910 | |||
| 1911 | Returns |
||
| 1912 | ------- |
||
| 1913 | str |
||
| 1914 | description value |
||
| 1915 | """ |
||
| 1916 | with qdb.sql_connection.TRN: |
||
| 1917 | sql = """SELECT description |
||
| 1918 | FROM qiita.default_workflow |
||
| 1919 | WHERE default_workflow_id = %s""" |
||
| 1920 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1921 | return qdb.sql_connection.TRN.execute_fetchlast() |
||
| 1922 | |||
| 1923 | @description.setter |
||
| 1924 | def description(self, description): |
||
| 1925 | """Changes the description of the default workflow |
||
| 1926 | |||
| 1927 | Parameters |
||
| 1928 | ---------- |
||
| 1929 | description : str |
||
| 1930 | New description value |
||
| 1931 | """ |
||
| 1932 | sql = """UPDATE qiita.default_workflow SET description = %s |
||
| 1933 | WHERE default_workflow_id = %s""" |
||
| 1934 | qdb.sql_connection.perform_as_transaction(sql, [description, self._id]) |
||
| 1935 | |||
| 1936 | @property |
||
| 1937 | def data_type(self): |
||
| 1938 | """Retrieves all the data_types accepted by the default workflow |
||
| 1939 | |||
| 1940 | Returns |
||
| 1941 | ---------- |
||
| 1942 | list of str |
||
| 1943 | The data types |
||
| 1944 | """ |
||
| 1945 | with qdb.sql_connection.TRN: |
||
| 1946 | sql = """SELECT data_type |
||
| 1947 | FROM qiita.default_workflow_data_type |
||
| 1948 | LEFT JOIN qiita.data_type USING (data_type_id) |
||
| 1949 | WHERE default_workflow_id = %s""" |
||
| 1950 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1951 | return qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 1952 | |||
| 1953 | @property |
||
| 1954 | def artifact_type(self): |
||
| 1955 | """Retrieves artifact_type that the workflow can be applied to |
||
| 1956 | |||
| 1957 | Returns |
||
| 1958 | ---------- |
||
| 1959 | str |
||
| 1960 | The name of the artifact type this workflow can be applied to |
||
| 1961 | """ |
||
| 1962 | with qdb.sql_connection.TRN: |
||
| 1963 | sql = """SELECT artifact_type |
||
| 1964 | FROM qiita.artifact_type |
||
| 1965 | LEFT JOIN qiita.default_workflow USING (artifact_type_id) |
||
| 1966 | WHERE default_workflow_id = %s""" |
||
| 1967 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1968 | return qdb.sql_connection.TRN.execute_fetchflatten()[0] |
||
| 1969 | |||
| 1970 | @property |
||
| 1971 | def graph(self): |
||
| 1972 | """Returns the graph that represents the workflow |
||
| 1973 | |||
| 1974 | Returns |
||
| 1975 | ------- |
||
| 1976 | networkx.DiGraph |
||
| 1977 | The graph representing the default workflow. |
||
| 1978 | """ |
||
| 1979 | g = nx.DiGraph() |
||
| 1980 | with qdb.sql_connection.TRN: |
||
| 1981 | # Retrieve all graph workflow nodes |
||
| 1982 | sql = """SELECT default_workflow_node_id |
||
| 1983 | FROM qiita.default_workflow_node |
||
| 1984 | WHERE default_workflow_id = %s |
||
| 1985 | ORDER BY default_workflow_node_id""" |
||
| 1986 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 1987 | db_nodes = qdb.sql_connection.TRN.execute_fetchflatten() |
||
| 1988 | |||
| 1989 | nodes = {n_id: DefaultWorkflowNode(n_id) for n_id in db_nodes} |
||
| 1990 | |||
| 1991 | # Retrieve all graph edges |
||
| 1992 | sql = """SELECT DISTINCT default_workflow_edge_id, parent_id, |
||
| 1993 | child_id |
||
| 1994 | FROM qiita.default_workflow_edge e |
||
| 1995 | JOIN qiita.default_workflow_node n |
||
| 1996 | ON e.parent_id = n.default_workflow_node_id |
||
| 1997 | OR e.child_id = n.default_workflow_node_id |
||
| 1998 | WHERE default_workflow_id = %s |
||
| 1999 | ORDER BY default_workflow_edge_id""" |
||
| 2000 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 2001 | db_edges = qdb.sql_connection.TRN.execute_fetchindex() |
||
| 2002 | |||
| 2003 | # let's track what nodes are actually being used so if they do not |
||
| 2004 | # have an edge we still return them as part of the graph |
||
| 2005 | used_nodes = nodes.copy() |
||
| 2006 | for edge_id, p_id, c_id in db_edges: |
||
| 2007 | e = DefaultWorkflowEdge(edge_id) |
||
| 2008 | g.add_edge(nodes[p_id], nodes[c_id], connections=e) |
||
| 2009 | if p_id in used_nodes: |
||
| 2010 | del used_nodes[p_id] |
||
| 2011 | if c_id in used_nodes: |
||
| 2012 | del used_nodes[c_id] |
||
| 2013 | # adding the missing nodes |
||
| 2014 | for ms in used_nodes: |
||
| 2015 | g.add_node(nodes[ms]) |
||
| 2016 | |||
| 2017 | return g |
||
| 2018 | |||
| 2019 | @property |
||
| 2020 | def parameters(self): |
||
| 2021 | """Retrieves the parameters that the workflow can be applied to |
||
| 2022 | |||
| 2023 | Returns |
||
| 2024 | ---------- |
||
| 2025 | dict, dict |
||
| 2026 | The dictionary of valid key: value pairs given by the sample or |
||
| 2027 | the preparation info file |
||
| 2028 | """ |
||
| 2029 | with qdb.sql_connection.TRN: |
||
| 2030 | sql = """SELECT parameters |
||
| 2031 | FROM qiita.default_workflow |
||
| 2032 | WHERE default_workflow_id = %s""" |
||
| 2033 | qdb.sql_connection.TRN.add(sql, [self.id]) |
||
| 2034 | return qdb.sql_connection.TRN.execute_fetchflatten()[0] |
||
| 2035 | |||
| 2036 | @parameters.setter |
||
| 2037 | def parameters(self, values): |
||
| 2038 | """Sets the parameters that the workflow can be applied to |
||
| 2039 | |||
| 2040 | Parameters |
||
| 2041 | ---------- |
||
| 2042 | dict : {'sample': dict, 'prep': dict} |
||
| 2043 | dict of dict with the key: value pairs for the 'sample' and 'prep' |
||
| 2044 | info files |
||
| 2045 | |||
| 2046 | Raises |
||
| 2047 | ------ |
||
| 2048 | ValueError |
||
| 2049 | if the passed parameter is not a properly formated dict |
||
| 2050 | """ |
||
| 2051 | if not isinstance(values, dict) or \ |
||
| 2052 | set(values.keys()) != set(['prep', 'sample']): |
||
| 2053 | raise ValueError("Improper format for values, should be " |
||
| 2054 | "{'sample': dict, 'prep': dict} ") |
||
| 2055 | with qdb.sql_connection.TRN: |
||
| 2056 | sql = """UPDATE qiita.default_workflow |
||
| 2057 | SET parameters = %s |
||
| 2058 | WHERE default_workflow_id = %s""" |
||
| 2059 | qdb.sql_connection.perform_as_transaction( |
||
| 2060 | sql, [dumps(values), self._id]) |