137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294 | class Fetch:
"""
A fetch object that handles retrieving elements from the table expression.
:param expression: the QueryExpression object to fetch from.
"""
def __init__(self, expression):
self._expression = expression
def __call__(
self,
*attrs,
offset=None,
limit=None,
order_by=None,
format=None,
as_dict=None,
squeeze=False,
download_path="."
):
"""
Fetches the expression results from the database into an np.array or list of dictionaries and
unpacks blob attributes.
:param attrs: zero or more attributes to fetch. If not provided, the call will return all attributes of this
table. If provided, returns tuples with an entry for each attribute.
:param offset: the number of tuples to skip in the returned result
:param limit: the maximum number of tuples to return
:param order_by: a single attribute or the list of attributes to order the results. No ordering should be assumed
if order_by=None. To reverse the order, add DESC to the attribute name or names: e.g. ("age DESC",
"frequency") To order by primary key, use "KEY" or "KEY DESC"
:param format: Effective when as_dict=None and when attrs is empty None: default from config['fetch_format'] or
'array' if not configured "array": use numpy.key_array "frame": output pandas.DataFrame. .
:param as_dict: returns a list of dictionaries instead of a record array. Defaults to False for .fetch() and to
True for .fetch('KEY')
:param squeeze: if True, remove extra dimensions from arrays
:param download_path: for fetches that download data, e.g. attachments
:return: the contents of the table in the form of a structured numpy.array or a dict list
"""
if order_by is not None:
# if 'order_by' passed in a string, make into list
if isinstance(order_by, str):
order_by = [order_by]
# expand "KEY" or "KEY DESC"
order_by = list(
_flatten_attribute_list(self._expression.primary_key, order_by)
)
attrs_as_dict = as_dict and attrs
if attrs_as_dict:
# absorb KEY into attrs and prepare to return attributes as dict (issue #595)
if any(is_key(k) for k in attrs):
attrs = list(self._expression.primary_key) + [
a for a in attrs if a not in self._expression.primary_key
]
if as_dict is None:
as_dict = bool(attrs) # default to True for "KEY" and False otherwise
# format should not be specified with attrs or is_dict=True
if format is not None and (as_dict or attrs):
raise DataJointError(
"Cannot specify output format when as_dict=True or "
"when attributes are selected to be fetched separately."
)
if format not in {None, "array", "frame"}:
raise DataJointError(
"Fetch output format must be in "
'{{"array", "frame"}} but "{}" was given'.format(format)
)
if not (attrs or as_dict) and format is None:
format = config["fetch_format"] # default to array
if format not in {"array", "frame"}:
raise DataJointError(
'Invalid entry "{}" in datajoint.config["fetch_format"]: '
'use "array" or "frame"'.format(format)
)
if limit is None and offset is not None:
logger.warning(
"Offset set, but no limit. Setting limit to a large number. "
"Consider setting a limit explicitly."
)
limit = 8000000000 # just a very large number to effect no limit
get = partial(
_get,
self._expression.connection,
squeeze=squeeze,
download_path=download_path,
)
if attrs: # a list of attributes provided
attributes = [a for a in attrs if not is_key(a)]
ret = self._expression.proj(*attributes)
ret = ret.fetch(
offset=offset,
limit=limit,
order_by=order_by,
as_dict=False,
squeeze=squeeze,
download_path=download_path,
format="array",
)
if attrs_as_dict:
ret = [
{k: v for k, v in zip(ret.dtype.names, x) if k in attrs}
for x in ret
]
else:
return_values = [
list(
(to_dicts if as_dict else lambda x: x)(
ret[self._expression.primary_key]
)
)
if is_key(attribute)
else ret[attribute]
for attribute in attrs
]
ret = return_values[0] if len(attrs) == 1 else return_values
else: # fetch all attributes as a numpy.record_array or pandas.DataFrame
cur = self._expression.cursor(
as_dict=as_dict, limit=limit, offset=offset, order_by=order_by
)
heading = self._expression.heading
if as_dict:
ret = [
dict((name, get(heading[name], d[name])) for name in heading.names)
for d in cur
]
else:
ret = list(cur.fetchall())
record_type = (
heading.as_dtype
if not ret
else np.dtype(
[
(
name,
type(value),
) # use the first element to determine blob type
if heading[name].is_blob
and isinstance(value, numbers.Number)
else (name, heading.as_dtype[name])
for value, name in zip(ret[0], heading.as_dtype.names)
]
)
)
try:
ret = np.array(ret, dtype=record_type)
except Exception as e:
raise e
for name in heading:
# unpack blobs and externals
ret[name] = list(map(partial(get, heading[name]), ret[name]))
if format == "frame":
ret = pandas.DataFrame(ret).set_index(heading.primary_key)
return ret
|