Skip to content

Commit d8996a0

Browse files
asahi417asahi417
committed
pip
1 parent a5a37a7 commit d8996a0

12 files changed

Lines changed: 116 additions & 53 deletions

File tree

‎LICENSE.txt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2021 asahi417
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ Words tagged with O are outside of named entities and the I-XXX tag is used for
113113
named entity of type XXX. Whenever two entities of type XXX are immediately next to each other, the
114114
first word of the second entity will be tagged B-XXX in order to show that it starts another entity.
115115
The custom dataset should has `train.txt` and `valid.txt` file in a same folder.
116-
Please take a look [sample custom data](examples/custom_dataset_sample).
116+
Please take a look [sample custom data](asset/custom_dataset_sample).
117117

118118
### Model Finetuning
119119
Language model finetuning can be done with a few lines:

examples/custom_dataset_sample/train.txt renamed to asset/custom_dataset_sample/train.txt

File renamed without changes.

examples/custom_dataset_sample/valid.txt renamed to asset/custom_dataset_sample/valid.txt

File renamed without changes.

experiments/summary_baseline.py renamed to examples/tner_paper/summary_baseline.py

File renamed without changes.

experiments/summary_panx.py renamed to examples/tner_paper/summary_panx.py

File renamed without changes.

‎setup.cfg‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[metadata]
2+
description-file = README.md

‎setup.py‎

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,53 @@
11
from setuptools import setup, find_packages
22

3-
VERSION = '0.0.0'
4-
NAME = 'tner'
5-
IS_RELEASED = False
6-
7-
with open('README.md') as f:
3+
with open('README.md', 'r') as f:
84
readme = f.read()
95

106
setup(
11-
name=NAME,
12-
version=VERSION,
13-
description='A library for language model finetuning on named entity recognition and model evaluation over cross-domain datasets.',
7+
name='tner',
8+
packages=find_packages(exclude=["asset", "examples", "static", "templates", "tests"]),
9+
version='0.0.0',
10+
license='MIT',
11+
description='Transformer-based named entity recognition',
12+
url='https://github.com/asahi417/tner',
13+
download_url="https://github.com/asahi417/tner/archive/0.0.0.tar.gz",
14+
keywords=['ner', 'nlp', 'language-model'],
1415
long_description=readme,
16+
long_description_content_type="text/markdown",
1517
author='Asahi Ushio',
1618
author_email='asahi1992ushio@gmail.com',
17-
packages=find_packages(),
19+
classifiers=[
20+
'Development Status :: 4 - Beta', # Chose either "3 - Alpha", "4 - Beta" or "5 - Production/Stable" as the current state of your package
21+
'Intended Audience :: Developers', # Define that your audience are developers
22+
'Intended Audience :: Science/Research',
23+
'Topic :: Scientific/Engineering',
24+
'License :: OSI Approved :: MIT License', # Again, pick a license
25+
'Programming Language :: Python :: 3', #Specify which pyhton versions that you want to support
26+
],
1827
include_package_data=True,
28+
test_suite='tests',
1929
install_requires=[
20-
'Pillow>=7.1.0',
21-
'mecab-python3==0.996.5', # this version can only work
22-
'uvicorn==0.11.8',
23-
'jinja2==2.11.2',
24-
'aiofiles==0.5.0',
25-
'fastapi==0.61.0',
26-
'matplotlib==3.3.1',
27-
'toml',
28-
'tensorboard',
29-
'torch',
30-
'transformers',
31-
'seqeval',
32-
'segtok'
33-
]
34-
)
30+
'Pillow>=7.1.0',
31+
# 'mecab-python3==0.996.5', # this version can only work
32+
'mecab-python3', # this version can only work
33+
'uvicorn==0.11.8',
34+
'jinja2==2.11.2',
35+
'aiofiles==0.5.0',
36+
'fastapi==0.61.0',
37+
'matplotlib==3.3.1',
38+
'toml',
39+
'tensorboard',
40+
'torch',
41+
'transformers',
42+
'seqeval',
43+
'segtok'
44+
],
45+
python_requires='>=3.6',
46+
entry_points={
47+
'console_scripts': [
48+
'tner-train = tner_cl.train:main',
49+
'tner-test = tner_cl.test:main',
50+
'tner-predict = tner_cl.predict:main'
51+
],
52+
}
53+
)

‎tner/get_dataset.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -172,14 +172,27 @@ def get_dataset_ner_single(data_name: str = 'wnut2017',
172172
files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
173173
if not os.path.exists(data_path):
174174
os.makedirs(data_path, exist_ok=True)
175-
os.system('wget -O {0}/data.tar.gz https://github.com/swiseman/neighbor-tagging/raw/master/data.tar.gz'.
175+
os.system('wget -O {0}/data.tar.gz https://github.com/asahi417/neighbor-tagging/blob/master/data.tar.gz'.
176176
format(CACHE_DIR))
177177
os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
178178
for i in ['train', 'dev', 'test']:
179179
conll_formatting(
180180
file_token=os.path.join(CACHE_DIR, 'data/conll2003/conll2003-{}.words'.format(i)),
181181
file_tag=os.path.join(CACHE_DIR, 'data/conll2003/conll2003-{}.nertags'.format(i)),
182182
output_file=os.path.join(data_path, '{}.txt'.format(i)))
183+
elif data_name == 'ontonotes5':
184+
files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
185+
if not os.path.exists(data_path):
186+
raise ValueError('please download Ontonotes5 from https://catalog.ldc.upenn.edu/LDC2013T19')
187+
# os.makedirs(data_path, exist_ok=True)
188+
# os.system('wget -O {0}/data.tar.gz https://github.com/asahi417/neighbor-tagging/blob/master/data.tar.gz'.
189+
# format(CACHE_DIR))
190+
# os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
191+
# for i in ['train', 'dev', 'test']:
192+
# conll_formatting(
193+
# file_token=os.path.join(CACHE_DIR, 'data/onto/{}.words'.format(i)),
194+
# file_tag=os.path.join(CACHE_DIR, 'data/onto/{}.ner'.format(i)),
195+
# output_file=os.path.join(data_path, '{}.txt'.format(i)))
183196
elif data_name == 'bc5cdr':
184197
files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
185198
if not os.path.exists(data_path):
@@ -229,7 +242,6 @@ def convert_to_iob(path, export):
229242
convert_to_iob('CDR_DevelopmentSet.PubTator.txt', 'dev.txt')
230243
convert_to_iob('CDR_TestSet.PubTator.txt', 'test.txt')
231244
convert_to_iob('CDR_TrainingSet.PubTator.txt', 'train.txt')
232-
233245
elif data_name == 'bionlp2004': # https://www.aclweb.org/anthology/W04-1213.pdf
234246
files_info = {'train': 'Genia4ERtask1.iob2', 'valid': 'Genia4EReval1.iob2'}
235247
if not os.path.exists(data_path):
@@ -261,18 +273,6 @@ def convert_to_iob(path, export):
261273
os.system('wget -O {0} https://groups.csail.mit.edu/sls/downloads/restaurant/restauranttest.bio'.format(
262274
os.path.join(data_path, 'valid.txt')))
263275
entity_first = True
264-
elif data_name == 'ontonotes5':
265-
files_info = {'train': 'train.txt', 'valid': 'dev.txt', 'test': 'test.txt'}
266-
if not os.path.exists(data_path):
267-
os.makedirs(data_path, exist_ok=True)
268-
os.system('wget -O {0}/data.tar.gz https://github.com/swiseman/neighbor-tagging/raw/master/data.tar.gz'.
269-
format(CACHE_DIR))
270-
os.system('tar -xzf {0}/data.tar.gz -C {0}'.format(CACHE_DIR))
271-
for i in ['train', 'dev', 'test']:
272-
conll_formatting(
273-
file_token=os.path.join(CACHE_DIR, 'data/onto/{}.words'.format(i)),
274-
file_tag=os.path.join(CACHE_DIR, 'data/onto/{}.ner'.format(i)),
275-
output_file=os.path.join(data_path, '{}.txt'.format(i)))
276276
elif data_name == 'mit_movie_trivia':
277277
files_info = {'train': 'train.txt', 'valid': 'valid.txt'}
278278
if not os.path.exists(data_path):

examples/example_inference.py renamed to tner_cl/predict.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ def get_options():
1111
return parser.parse_args()
1212

1313

14-
if __name__ == '__main__':
14+
def main():
1515
opt = get_options()
1616
classifier = TransformersNER(checkpoint=opt.checkpoint)
1717
test_sentences = [

0 commit comments

Comments
 (0)