kj7rrv/gptc
1
0
Fork 0
Code Issues 2 Pull Requests Projects Releases 12 Wiki Activity

Make pack a function

Browse Source
This commit is contained in:
Samuel Sloniker 2022-05-21 13:09:53 -07:00
parent bebd286163
commit e06f2def24
1 changed files with 31 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
utils/pack.py
View File

@ -4,14 +4,13 @@ import sys
import os import os
import json import json
if len(sys.argv) != 2:
print("usage: pack.py <path>", file=sys.stderr)
exit(1)
paths = os.listdir(sys.argv[1]) def pack(directory, print_exceptions=True):
texts = {} paths = os.listdir(directory)
texts = {}
exceptions = []
for path in paths: for path in paths:
texts[path] = [] texts[path] = []
try: try:
for file in os.listdir(os.path.join(sys.argv[1], path)): for file in os.listdir(os.path.join(sys.argv[1], path)):
@ -19,13 +18,24 @@ for path in paths:
with open(os.path.join(sys.argv[1], path, file)) as f: with open(os.path.join(sys.argv[1], path, file)) as f:
texts[path].append(f.read()) texts[path].append(f.read())
except Exception as e: except Exception as e:
exceptions.append((e,))
if print_exceptions:
print(e, file=sys.stderr) print(e, file=sys.stderr)
except Exception as e: except Exception as e:
exceptions.append((e,))
if print_exceptions:
print(e, file=sys.stderr) print(e, file=sys.stderr)
raw_model = [] raw_model = []
for category, cat_texts in texts.items(): for category, cat_texts in texts.items():
raw_model += [{"category": category, "text": i} for i in cat_texts] raw_model += [{"category": category, "text": i} for i in cat_texts]
print(json.dumps(raw_model)) return raw_model, exceptions
if len(sys.argv) != 2:
print("usage: pack.py <path>", file=sys.stderr)
exit(1)
print(json.dumps(pack(sys.argv[1])[0]))