Download Data

Download Data#

For the tutorials in this book, we will use partially-preprocessed data from two open multi-echo datasets: Euskalibur and Cambridge. For more information about these datasets, see Open Multi-Echo Datasets.

import os

from datalad import api as dapi

DATA_DIR = os.path.abspath('../data')

# Download PAFIN fMRIPrep data
dset_dir = os.path.join(DATA_DIR, 'ds006185')
os.makedirs(dset_dir, exist_ok=True)
dapi.install(
    path=dset_dir,
    source='https://github.com/OpenNeuroDatasets/ds006185.git',
)
dapi.get(os.path.join(dset_dir, 'sub-24053', 'ses-1', 'func', 'sub-24053_ses-1_task-rat_rec-nordic_*'), recursive=True)
dapi.get(os.path.join(dset_dir, 'sub-24053', 'ses-1', 'anat', 'sub-24053_ses-1_*'), recursive=True)

For now, we will use the Datalab API to download some data we’re storing on OpenNeuro.

import os
from pathlib import Path

from datalad import api as dapi

DATA_DIR = os.path.abspath('../data')

# Download PAFIN fMRIPrep data
dset_dir = os.path.join(DATA_DIR, 'ds006185')
os.makedirs(dset_dir, exist_ok=True)
dapi.install(
    path=dset_dir,
    source='https://github.com/OpenNeuroDatasets/ds006185.git',
)
subj_dir = os.path.join(dset_dir, 'sub-24053', 'ses-1')
func_dir = Path(os.path.join(subj_dir, 'func'))
func_files = list(func_dir.glob('sub-24053_ses-1_task-rat_rec-nordic_*'))
for f in func_files:
    dapi.get(f)

anat_dir = Path(os.path.join(subj_dir, 'anat'))
anat_files = list(anat_dir.glob('*'))
for f in anat_files:
    dapi.get(f)

Hide code cell output

 
It is highly recommended to configure Git before using DataLad. Set both 'user.name' and 'user.email' configuration variables.

[INFO] Attempting a clone into /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185 

[INFO] Attempting to clone from https://github.com/OpenNeuroDatasets/ds006185.git to /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185 

[INFO] Start enumerating objects 

[INFO] Start counting objects 

[INFO] Start compressing objects 

[INFO] Start receiving objects 

[INFO] Start resolving deltas 

[INFO] Completed clone attempts for Dataset(/home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185) 

[INFO] Remote origin not usable by git-annex; setting annex-ignore 

[INFO] https://github.com/OpenNeuroDatasets/ds006185.git/config download failed: Not Found 

[INFO] Cannot run git-annex-remote-openneuro -- It is not installed in PATH (/usr/lib/git-core:/opt/hostedtoolcache/Python/3.12.11/x64/bin:/opt/hostedtoolcache/Python/3.12.11/x64:/snap/bin:/home/runner/.local/bin:/opt/pipx_bin:/home/runner/.cargo/bin:/home/runner/.config/composer/vendor/bin:/usr/local/.ghcup/bin:/home/runner/.dotnet/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin) 

[INFO] Cannot run git-annex-remote-openneuro -- It is not installed in PATH (/usr/lib/git-core:/opt/hostedtoolcache/Python/3.12.11/x64/bin:/opt/hostedtoolcache/Python/3.12.11/x64:/snap/bin:/home/runner/.local/bin:/opt/pipx_bin:/home/runner/.cargo/bin:/home/runner/.config/composer/vendor/bin:/usr/local/.ghcup/bin:/home/runner/.dotnet/tools:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin) 

[INFO] unable to use external special remote git-annex-remote-openneuro 

[INFO] Author identity unknown
| 
| *** Please tell me who you are.
| 
| Run
| 
|   git config --global user.email "you@example.com"
|   git config --global user.name "Your Name"
| 
| to set your account's default identity.
| Omit --global to set the identity only in this repository.
| 
| fatal: unable to auto-detect email address (got 'runner@runnervmwhb2z.(none)') 

[INFO] git-annex: failed to read sha from git commit-tree 

install(error): /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185 (dataset) [CommandError: 'git -c diff.ignoreSubmodules=none -c core.quotepath=false annex init -c annex.dotfiles=true' failed with exitcode 1 under /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185] [CommandError: 'git -c diff.ignoreSubmodules=none -c core.quotepath=false annex init -c annex.dotfiles=true' failed with exitcode 1 under /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185]

---------------------------------------------------------------------------
IncompleteResultsError                    Traceback (most recent call last)
Cell In[1], line 11
      9 dset_dir = os.path.join(DATA_DIR, 'ds006185')
     10 os.makedirs(dset_dir, exist_ok=True)
---> 11 dapi.install(
     12     path=dset_dir,
     13     source='https://github.com/OpenNeuroDatasets/ds006185.git',
     14 )
     15 subj_dir = os.path.join(dset_dir, 'sub-24053', 'ses-1')
     16 func_dir = Path(os.path.join(subj_dir, 'func'))

File /opt/hostedtoolcache/Python/3.12.11/x64/lib/python3.12/site-packages/datalad/interface/base.py:772, in eval_results.<locals>.eval_func(*args, **kwargs)
    767         return results
    769 lgr.log(2,
    770         "Returning return_func from eval_func for %s",
    771         wrapped_class)
--> 772 return return_func(*args, **kwargs)

File /opt/hostedtoolcache/Python/3.12.11/x64/lib/python3.12/site-packages/datalad/interface/base.py:762, in eval_results.<locals>.eval_func.<locals>.return_func(*args_, **kwargs_)
    752 results = _execute_command_(
    753     interface=wrapped_class,
    754     cmd=wrapped,
   (...)    757     exec_kwargs=common_params,
    758 )
    759 if inspect.isgenerator(results):
    760     # unwind generator if there is one, this actually runs
    761     # any processing
--> 762     results = list(results)
    763 if return_type == 'item-or-list' and \
    764         len(results) < 2:
    765     return results[0] if results else None

File /opt/hostedtoolcache/Python/3.12.11/x64/lib/python3.12/site-packages/datalad/interface/base.py:939, in _execute_command_(interface, cmd, cmd_args, cmd_kwargs, exec_kwargs)
    936     render_action_summary(action_summary)
    938 if incomplete_results:
--> 939     raise IncompleteResultsError(
    940         failed=incomplete_results,
    941         msg="Command did not complete successfully")

IncompleteResultsError: Command did not complete successfully. 1 failed:
[{'action': 'install',
  'error_message': "CommandError: 'git -c diff.ignoreSubmodules=none -c "
                   "core.quotepath=false annex init -c annex.dotfiles=true' "
                   'failed with exitcode 1 under '
                   '/home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185',
  'exception': CommandError: 'git -c diff.ignoreSubmodules=none -c core.quotepath=false annex init -c annex.dotfiles=true' failed with exitcode 1 under /home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185 [clone.py:clone_dataset:521,clone.py:_post_gitclone_processing_:589,clone.py:_annex_init:700,annexrepo.py:_init:1539,annexrepo.py:_call_annex:996,runner.py:run:243],
  'exception_traceback': '[clone.py:clone_dataset:521,clone.py:_post_gitclone_processing_:589,clone.py:_annex_init:700,annexrepo.py:_init:1539,annexrepo.py:_call_annex:996,runner.py:run:243]',
  'message': "CommandError: 'git -c diff.ignoreSubmodules=none -c "
             "core.quotepath=false annex init -c annex.dotfiles=true' failed "
             'with exitcode 1 under '
             '/home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185',
  'path': '/home/runner/work/multi-echo-data-analysis/multi-echo-data-analysis/data/ds006185',
  'refds': None,
  'source': {'default_destpath': 'ds006185',
             'giturl': 'https://github.com/OpenNeuroDatasets/ds006185.git',
             'source': 'https://github.com/OpenNeuroDatasets/ds006185.git',
             'type': 'giturl',
             'version': None},
  'source_url': 'https://github.com/OpenNeuroDatasets/ds006185.git',
  'status': 'error',
  'type': 'dataset'}]