Skip to content

Commit 8279b46

Browse files
xiulirenxiuliren
committed
semantic training
1 parent 93fc8c2 commit 8279b46

12 files changed

Lines changed: 663 additions & 20 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ Neuron segmentation and synapse detection using PyTorch
1111
# Features
1212
- [x] Training using whole terabyte or even petabyte of image volume.
1313
- [x] Training using multiple version of image datasets as data augmentation.
14+
- [x] Data augmentation without zero-filling.
1415

1516
# Install
1617
python setup.py install

‎environment.yml‎

Lines changed: 286 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,286 @@
1+
name: pytorch
2+
channels:
3+
- pytorch
4+
- nvidia
5+
- conda-forge
6+
- defaults
7+
dependencies:
8+
- _libgcc_mutex=0.1=conda_forge
9+
- _openmp_mutex=4.5=1_gnu
10+
- backcall=0.2.0=pyhd3eb1b0_0
11+
- blas=1.0=mkl
12+
- blas-devel=3.9.0=11_linux64_mkl
13+
- bzip2=1.0.8=h7b6447c_0
14+
- c-ares=1.18.1=h7f8727e_0
15+
- ca-certificates=2021.10.26=h06a4308_2
16+
- certifi=2021.10.8=py38h06a4308_2
17+
- charset-normalizer=2.0.4=pyhd3eb1b0_0
18+
- conda=4.11.0=py38h578d9bd_0
19+
- conda-package-handling=1.7.3=py38h27cfd23_1
20+
- cudatoolkit=11.1.74=h6bb024c_0
21+
- ffmpeg=4.2.2=h20bf706_0
22+
- freetype=2.11.0=h70c0345_0
23+
- giflib=5.2.1=h7b6447c_0
24+
- gmp=6.2.1=h2531618_2
25+
- gnutls=3.6.15=he1e5248_0
26+
- icu=58.2=he6710b0_3
27+
- intel-openmp=2021.4.0=h06a4308_3561
28+
- ipython=7.26.0=py38hb070fc8_0
29+
- jedi=0.18.0=py38h06a4308_1
30+
- jpeg=9b=h024ee3a_2
31+
- krb5=1.19.2=hac12032_0
32+
- lame=3.100=h7b6447c_0
33+
- lcms2=2.12=h3be6417_0
34+
- ld_impl_linux-64=2.35.1=h7274673_9
35+
- libarchive=3.4.2=h62408e4_0
36+
- libblas=3.9.0=11_linux64_mkl
37+
- libcblas=3.9.0=11_linux64_mkl
38+
- libcurl=7.80.0=h0b77cf5_0
39+
- libedit=3.1.20210910=h7f8727e_0
40+
- libev=4.33=h7f8727e_1
41+
- libffi=3.3=he6710b0_2
42+
- libgcc-ng=11.2.0=h1d223b6_11
43+
- libgomp=11.2.0=h1d223b6_11
44+
- libidn2=2.3.2=h7f8727e_0
45+
- liblapack=3.9.0=11_linux64_mkl
46+
- liblapacke=3.9.0=11_linux64_mkl
47+
- libnghttp2=1.46.0=hce63b2e_0
48+
- libopus=1.3.1=h7b6447c_0
49+
- libpng=1.6.37=hbc83047_0
50+
- libsolv=0.7.16=h8b12597_0
51+
- libssh2=1.9.0=h1ba5d50_1
52+
- libstdcxx-ng=11.2.0=he4da1e4_11
53+
- libtasn1=4.16.0=h27cfd23_0
54+
- libtiff=4.2.0=h85742a9_0
55+
- libunistring=0.9.10=h27cfd23_0
56+
- libuv=1.40.0=h7b6447c_0
57+
- libvpx=1.7.0=h439df22_0
58+
- libwebp=1.2.0=h89dd481_0
59+
- libwebp-base=1.2.0=h27cfd23_0
60+
- libxml2=2.9.12=h03d6c58_0
61+
- lz4-c=1.9.3=h295c915_1
62+
- mamba=0.7.3=py38h9709c9f_0
63+
- matplotlib-inline=0.1.2=pyhd3eb1b0_2
64+
- mkl=2021.3.0=h06a4308_520
65+
- mkl-devel=2021.3.0=h66538d2_520
66+
- mkl-include=2021.3.0=h06a4308_520
67+
- mkl-service=2.4.0=py38h7f8727e_0
68+
- mkl_fft=1.3.1=py38hd3c417c_0
69+
- mkl_random=1.2.2=py38h51133e4_0
70+
- ncurses=6.3=h7f8727e_2
71+
- nettle=3.7.3=hbbd107a_1
72+
- ninja=1.10.2=py38hd09550d_3
73+
- olefile=0.46=pyhd3eb1b0_0
74+
- openh264=2.1.1=h4ff587b_0
75+
- openssl=1.1.1m=h7f8727e_0
76+
- parso=0.8.3=pyhd3eb1b0_0
77+
- pexpect=4.8.0=pyhd3eb1b0_3
78+
- pickleshare=0.7.5=pyhd3eb1b0_1003
79+
- pip=21.2.4=py38h06a4308_0
80+
- prompt-toolkit=3.0.20=pyhd3eb1b0_0
81+
- ptyprocess=0.7.0=pyhd3eb1b0_2
82+
- pybind11=2.8.1=py38hd09550d_0
83+
- pycosat=0.6.3=py38h7b6447c_1
84+
- pygments=2.10.0=pyhd3eb1b0_0
85+
- pysocks=1.7.1=py38h06a4308_0
86+
- python=3.8.12=h12debd9_0
87+
- python_abi=3.8=2_cp38
88+
- pytorch=1.9.1=py3.8_cuda11.1_cudnn8.0.5_0
89+
- readline=8.1.2=h7f8727e_1
90+
- reproc=14.2.1=h36c2ea0_0
91+
- reproc-cpp=14.2.1=h58526e2_0
92+
- ruamel_yaml=0.15.100=py38h27cfd23_0
93+
- setuptools=58.0.4=py38h06a4308_0
94+
- sqlite=3.37.0=hc218d9a_0
95+
- tk=8.6.11=h1ccaba5_0
96+
- torchaudio=0.9.1=py38
97+
- traitlets=5.1.1=pyhd3eb1b0_0
98+
- typing_extensions=3.10.0.2=pyh06a4308_0
99+
- wcwidth=0.2.5=pyhd3eb1b0_0
100+
- x264=1!157.20191217=h7b6447c_0
101+
- xtensor=0.24.0=h4bd325d_0
102+
- xtensor-blas=0.20.0=h4bd325d_0
103+
- xtensor-python=0.26.0=py38h2b96118_1
104+
- xtl=0.7.4=h4bd325d_0
105+
- xz=5.2.5=h7b6447c_0
106+
- yaml=0.2.5=h7b6447c_0
107+
- zlib=1.2.11=h7f8727e_4
108+
- zstd=1.4.9=haebb681_0
109+
- pip:
110+
- absl-py==0.12.0
111+
- argon2-cffi==21.3.0
112+
- astunparse==1.6.3
113+
- attrs==21.1.0
114+
- awscli==1.19.72
115+
- bleach==4.1.0
116+
- boto3==1.17.68
117+
- botocore==1.20.112
118+
- brotli==1.0.9
119+
- brotlipy==0.7.0
120+
- cachetools==4.2.2
121+
- cffi==1.14.5
122+
- chardet==4.0.0
123+
- click==8.1.3
124+
- cloud-files==4.9.1
125+
- cloud-volume==8.8.1
126+
- colorama==0.4.3
127+
- compressed-segmentation==2.2.0
128+
- compresso==3.0.0
129+
- connected-components-3d==3.2.0
130+
- crc32c==2.2
131+
- cryptography==3.4.7
132+
- cycler==0.10.0
133+
- cython==0.29.21
134+
- decorator==4.4.2
135+
- deflate==0.3.0
136+
- defusedxml==0.7.1
137+
- deprecated==1.2.12
138+
- detect-secrets==1.1.0
139+
- dijkstra3d==1.9.1
140+
- dill==0.3.3
141+
- docutils==0.15.2
142+
- dracopy==1.1.1
143+
- edt==2.1.0
144+
- einops==0.4.0
145+
- entrypoints==0.3
146+
- fasteners==0.16
147+
- fastremap==1.13.2
148+
- fill-voids==2.0.1
149+
- flatbuffers==2.0
150+
- fpzip==1.1.4
151+
- fvcore==0.1.5.post20220512
152+
- gast==0.5.3
153+
- gevent==21.1.2
154+
- google-api-core==1.26.3
155+
- google-apitools==0.5.32
156+
- google-auth==1.30.0
157+
- google-auth-oauthlib==0.4.4
158+
- google-cloud-core==1.6.0
159+
- google-cloud-storage==1.38.0
160+
- google-crc32c==1.1.2
161+
- google-pasta==0.2.0
162+
- google-resumable-media==1.2.0
163+
- googleapis-common-protos==1.53.0
164+
- greenlet==1.1.0
165+
- grpcio==1.37.1
166+
- h5py==3.2.1
167+
- httplib2==0.19.1
168+
- humanize==3.5.0
169+
- idna==2.10
170+
- imageio==2.9.0
171+
- inflection==0.5.1
172+
- iniconfig==1.1.1
173+
- iopath==0.1.10
174+
- ipykernel==6.7.0
175+
- ipywidgets==8.0.0b1
176+
- jinja2==3.0.3
177+
- jmespath==0.10.0
178+
- joblib==1.0.1
179+
- json5==0.9.5
180+
- jsonschema==3.2.0
181+
- jupyter==1.0.0
182+
- jupyter-client==7.1.2
183+
- jupyter-console==6.4.0
184+
- jupyter-core==4.9.1
185+
- jupyterlab-pygments==0.1.2
186+
- jupyterlab-widgets==2.0.0b1
187+
- keras==2.8.0rc1
188+
- keras-preprocessing==1.1.2
189+
- kimimaro==2.1.1
190+
- kiwisolver==1.3.1
191+
- libclang==12.0.0
192+
- markdown==3.3.4
193+
- matplotlib==3.4.1
194+
- mistune==0.8.4
195+
- multiprocess==0.70.11.1
196+
- nbclient==0.5.10
197+
- nbconvert==6.4.0
198+
- nbformat==5.1.3
199+
- nest-asyncio==1.5.4
200+
- networkx==2.5.1
201+
- neuroglancer==2.29
202+
- nibabel==3.2.1
203+
- nose==1.3.7
204+
- notebook==6.4.7
205+
- numpy==1.20.2
206+
- oauth2client==4.1.3
207+
- oauthlib==3.1.0
208+
- opencv-python==4.5.1.48
209+
- opt-einsum==3.3.0
210+
- orjson==3.5.2
211+
- packaging==20.9
212+
- pandas==1.4.3
213+
- pandocfilters==1.5.0
214+
- pathos==0.2.7
215+
- pillow==8.2.0
216+
- pluggy==0.13.1
217+
- portalocker==2.5.1
218+
- posix-ipc==1.0.5
219+
- pox==0.2.9
220+
- ppft==1.6.6.3
221+
- prometheus-client==0.12.0
222+
- protobuf==3.15.8
223+
- psutil==5.4.3
224+
- py==1.10.0
225+
- pyasn1==0.4.8
226+
- pyasn1-modules==0.2.8
227+
- pycparser==2.20
228+
- pynrrd==0.4.2
229+
- pyopenssl==20.0.1
230+
- pyparsing==2.4.7
231+
- pyrsistent==0.17.3
232+
- pysimdjson==3.2.0
233+
- pyspng-seunglab==1.0.0
234+
- pytest==6.2.4
235+
- python-dateutil==2.8.1
236+
- python-jsonschema-objects==0.3.14
237+
- pytz==2021.1
238+
- pywavelets==1.1.1
239+
- pyyaml==5.4.1
240+
- pyzmq==22.3.0
241+
- qtconsole==5.2.2
242+
- qtpy==2.0.0
243+
- requests==2.25.1
244+
- requests-oauthlib==1.3.0
245+
- rsa==4.7.2
246+
- s3transfer==0.4.0
247+
- scikit-image==0.18.1
248+
- scikit-learn==0.24.2
249+
- scipy==1.6.3
250+
- send2trash==1.8.1b0
251+
- simpleitk==1.2.4
252+
- simplejpeg==1.6.0
253+
- six==1.15.0
254+
- sockjs-tornado==1.0.7
255+
- tabulate==0.8.10
256+
- tenacity==7.0.0
257+
- tensorboard==2.8.0
258+
- tensorboard-data-server==0.6.0
259+
- tensorboard-plugin-wit==1.8.0
260+
- tensorflow==2.8.0rc1
261+
- tensorflow-io-gcs-filesystem==0.23.1
262+
- terminado==0.12.1
263+
- testpath==0.5.0
264+
- tf-estimator-nightly==2.8.0.dev2021122109
265+
- threadpoolctl==2.1.0
266+
- tifffile==2021.4.8
267+
- tinybrain==1.2.1
268+
- toml==0.10.2
269+
- torch==1.8.1
270+
- torch-tb-profiler==0.2.1
271+
- torchio==0.18.36
272+
- torchvision==0.9.1
273+
- tornado==6.1
274+
- tqdm==4.64.0
275+
- typing-extensions==3.7.4.3
276+
- urllib3==1.26.4
277+
- werkzeug==1.0.1
278+
- wheel==0.37.1
279+
- widgetsnbextension==4.0.0b1
280+
- wrapt==1.12.1
281+
- yacs==0.1.8
282+
- zmesh==0.5.0
283+
- zope-event==4.5.0
284+
- zope-interface==5.4.0
285+
- zstandard==0.15.2
286+
prefix: /mnt/home/jwu/anaconda3/envs/pytorch

‎neutorch/dataset/base.py‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from abc import abstractproperty
12
from typing import Union
23
from functools import cached_property
34
import math
@@ -24,6 +25,12 @@ def worker_init_fn(worker_id: int):
2425
dataset.start = overall_start + worker_id * per_worker
2526
dataset.end = min(dataset.start + per_worker, overall_end)
2627

28+
def path_to_dataset_name(path: str, dataset_names: list):
29+
for dataset_name in dataset_names:
30+
if dataset_name in path:
31+
return dataset_name
32+
33+
2734

2835
class DatasetBase(torch.utils.data.IterableDataset):
2936
def __init__(self,
@@ -48,8 +55,10 @@ def __init__(self,
4855
self.transform.shrink_size[:3] + \
4956
self.transform.shrink_size[-3:]
5057

51-
# inherite this class and build the samples
52-
self.samples = None
58+
@cached_property
59+
@abstractproperty
60+
def samples(self):
61+
pass
5362

5463
@cached_property
5564
def sample_num(self):

0 commit comments

Comments
 (0)