From ae054abb9683a5f95c459d48425aaf5bfffb5f0d Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Tue, 22 Sep 2026 12:47:16 -0500 Subject: [PATCH 1/5] added method for story file cleaning with custom archives --- 04-Rename-Tags.py | 1 - README.md | 9 +++++ example.yml | 10 +++++ requirements.txt | 2 +- shared_python/Args.py | 23 ++++++++++++ shared_python/Common.py | 12 +++++- story_cleanup/keywords.py | 30 +++++++++++++++ story_cleanup/messages.py | 36 ++++++++++++++++++ story_cleanup/outputs.py | 30 +++++++++++++++ story_cleanup/parsing.py | 38 +++++++++++++++++++ xx-Extract-Notes-Summaries-Other.py | 57 +++++++++++++++++++++++++++++ 11 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 story_cleanup/keywords.py create mode 100644 story_cleanup/messages.py create mode 100644 story_cleanup/outputs.py create mode 100644 story_cleanup/parsing.py create mode 100644 xx-Extract-Notes-Summaries-Other.py diff --git a/04-Rename-Tags.py b/04-Rename-Tags.py index 2316ca5..a0f90a0 100755 --- a/04-Rename-Tags.py +++ b/04-Rename-Tags.py @@ -22,7 +22,6 @@ tw_tags = list(csv.DictReader(csvfile)) tag_headers = tags.tag_export_map total = len(tw_tags) - for cur, row in enumerate(tw_tags): tags.update_tag_row(row) print_progress(cur, total, "tags") diff --git a/README.md b/README.md index 92eebe3..536a052 100755 --- a/README.md +++ b/README.md @@ -255,6 +255,15 @@ On it we can either type (and press enter): Given a comma-separated list of story ids specified in the `story_ids_to_remove` parameter, deletes the corresponding rows from the stories table in the final output database. +## Extract notes, summaries, and commercial links from chapters + +Extracts notes, summaries, content warnings, and + +This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. + +note that this does NOT load chapters into the stories table + + ## Parameters diff --git a/example.yml b/example.yml index 25e0e67..82758ef 100755 --- a/example.yml +++ b/example.yml @@ -28,3 +28,13 @@ bookmark_ids_to_remove: /path/to/ids to remove.txt # Chapter file location chapters_path: /path/to/chapter/files/stories chapters_file_extensions: txt, html + + + +# options for story cleanup +# leave blank to ignore story files; any value here will remove flagged text from the original +remove_option: +chapters_backup_path: /path/to/backup/chapter/story/output/files +output_csv: /path/to/output/Desired_CSV_name.csv +# Comma-separated list of types to scan for +scan_types: [notes, warnings, summary, commercial] diff --git a/requirements.txt b/requirements.txt index fd12516..476d90d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ charade==1.0.3 Genshi==0.7.7 ipaddr==2.2.0 keyring==24.2.0 -lxml==4.9.3 +lxml==6.1.3 ndg_httpsclient==0.5.1 pyasn1==0.5.0 pytest==7.4.3 diff --git a/shared_python/Args.py b/shared_python/Args.py index b543b99..59e12d1 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -269,3 +269,26 @@ def args_for_08(self): ) self._print_args(self.args) return self.args + + def args_for_sn_extraction(self): + if self.args.chapters_path is None: + self.args.chapters_path = input( + "Location of the text files containing the stories:" + ) + if self.args.remove_option is not None: + if self.args.chapters_backup_path is None: + self.args.chapters_backup_path = input( + "Desired location (full path) of the original text backup" + ) + if os.path.exists(self.args.chapters_backup_path): + os.rmdir(self.args.chapters_backup_path) + + if self.args.output_csv is None: + self.args.output_csv = input( + "Output path for the extracted CSV" + ) + if self.args.scan_types is None: + self.args.scan_types = ['notes', 'warnings', 'summary', 'commercial'] + + self._print_args(self.args) + return self.args \ No newline at end of file diff --git a/shared_python/Common.py b/shared_python/Common.py index 1214162..fb0afbd 100755 --- a/shared_python/Common.py +++ b/shared_python/Common.py @@ -1,6 +1,7 @@ # -- coding: utf-8 -- -import sys from importlib import reload +import os +import sys reload(sys) # sys.setdefaultencoding('utf8') #setdefaultencoding is disabled in Python 3. UTF-8 is also default coding. @@ -13,3 +14,12 @@ def print_progress(cur, total, prog_type="stories"): sys.stdout.write("\r{0}/{1} {2}".format(cur, total, prog_type)) sys.stdout.flush() return cur + +def recursive_story_listdir(main_path): + storyfiles = [x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name] + subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()] + for dir in subdirs: + stories = [x for x in os.listdir(os.path.join(main_path, dir)) if "html" in x or "txt" in x] + for s in stories: + storyfiles.append(os.path.join(dir, s)) + return storyfiles diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py new file mode 100644 index 0000000..27612eb --- /dev/null +++ b/story_cleanup/keywords.py @@ -0,0 +1,30 @@ +class Keywords: + def __init__(self): + self.notes = ["note", "Note", "notes", "Notes", "A/n", "a/n", "A/N", "disclaimer", "Disclaimer", "DISCLAIMER"] + self.warnings = ["warning", "Warning", "warnings", "Warnings", "cw", "CW", "content warning", "Content Warning", "trigger warning", + "Trigger Warning", "Content warning", "Trigger warning", "tw", "TW"] + self.summary = ["summary", "Summary", "summaries", "Summaries", "prompt", "Prompt"] + self.commercial = ["Patreon", "patreon", "mailto", "Ko-fi", "ko-fi", "Ko-Fi", "gofundme", "GoFundMe"] + # self.ratings + # self.wordcount + # self.pairings + # self.characters + # others defined + def detect(self, type, text): + match type: + case "notes": + keywords = self.notes + case "warnings": + keywords = self.warnings + case "summary": + keywords = self.summary + case "commercial": + keywords = self.commercial + case _: + raise NameError( + "unsupported type: must be 'notes', 'warnings', 'summary', or 'commercial'; check config file" + ) + for k in keywords: + if k in text: + return True + return False diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py new file mode 100644 index 0000000..3bce060 --- /dev/null +++ b/story_cleanup/messages.py @@ -0,0 +1,36 @@ +from colorama import Fore, Style + +class Messaging: + ''' + class that handles interactivity, message colors, etc + ''' + def __init__(self, text): + self.text = text + self.summary_color = Fore.BLUE + self.notes_color = Fore.MAGENTA + self.warning_color = Fore.LIGHTBLUE_EX + self.commercial_color = Fore.RED + self.additional_lines_color = Fore.LIGHTBLACK_EX + self.general = Fore.GREEN + self.reset = Style.RESET_ALL + def check_init_response(self, type): + match type: + case "summary": + print(self.summary_color + f"{type}:\n" + self.reset + self.text) + case "notes": + print(self.notes_color + f"{type}:\n" + self.reset + self.text) + case "warnings": + print(self.warning_color + f"{type}:\n" + self.reset + self.text) + case "commercial": + print(self.commercial_color + f"{type}:\n" + self.reset + self.text) + r = input(self.general + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + self.reset) + if r.lower() == "n": + return False + return True + def check_additional_lines(self, preview): + print(f"{self.text}\n") + print(self.additional_lines_color + preview + self.reset) + r = input((self.general + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + self.reset)) + if r == "n": + return True + return False \ No newline at end of file diff --git a/story_cleanup/outputs.py b/story_cleanup/outputs.py new file mode 100644 index 0000000..cf5a79a --- /dev/null +++ b/story_cleanup/outputs.py @@ -0,0 +1,30 @@ +import csv +import os + + +class OutputCSV: + def __init__(self, filename: str, columns: list): + self.filename = filename + self.columns = columns + ['story_identifier'] + def init_csv(self): + with open(self.filename, "w", encoding="utf_8_sig", newline="") as fp: + csvf = csv.DictWriter(fp, fieldnames=self.columns) + csvf.writeheader() + def write_data(self, data: dict): + with open(self.filename, "a", encoding="utf_8_sig", newline="") as f: + csvf = csv.DictWriter(f, fieldnames=self.columns) + csvf.writerow(data) + +class OutputStoryFiles: + def __init__(self, output_path): + self.output_path = output_path + def strip_and_rewrite(self, text, options: dict): + text = "".join(text) + for o in options.keys(): + hit_lines = options[o].split("\n") + for h in hit_lines: + text = text.replace(h, "") + return text + def out_to_file(self, filename, strippedtext): + with open(os.path.join(self.output_path, filename), "w") as f: + f.write(strippedtext) diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py new file mode 100644 index 0000000..43ca135 --- /dev/null +++ b/story_cleanup/parsing.py @@ -0,0 +1,38 @@ +from .keywords import Keywords +from .messages import Messaging + +class Parser: + def __init__(self, textlines: list, options: list): + self.textlines = textlines + self.max = len(self.textlines) + self.options_dict = {} + for opt in options: + self.options_dict.update({opt:""}) + self.end_message = "***no more lines to preview, press enter to continue***" + self.kwords = Keywords() + + def detection_loop(self, type: str, i: int): + m = Messaging(self.textlines[i]) + if m.check_init_response(type): + local_count = i + 1 + txt = self.textlines[i] + try: + a = m.check_additional_lines(self.textlines[local_count]) + except IndexError: + input(self.end_message) + return txt + while a and local_count < self.max: + preview = self.textlines[local_count + 1] + txt = txt + self.textlines[local_count] + a = m.check_additional_lines(preview) + local_count += 1 + return txt + return "" + + def parse_lines(self): + # this is per story file + for i in range(0, int(self.max)): + for o in self.options_dict.keys(): + if self.kwords.detect(o, self.textlines[i]): + self.options_dict[o] = self.options_dict[o] + "\n" + self.detection_loop(o, i) + return self.options_dict diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py new file mode 100644 index 0000000..7497be0 --- /dev/null +++ b/xx-Extract-Notes-Summaries-Other.py @@ -0,0 +1,57 @@ +import os +import shutil + +from shared_python.Args import Args +from shared_python.Common import recursive_story_listdir + +from story_cleanup.outputs import OutputCSV, OutputStoryFiles +from story_cleanup.parsing import Parser + + +#TODO: improve logging +''' +For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them +note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) +''' + +if __name__ == "__main__": + args_obj = Args() + args = args_obj.args_for_sn_extraction() + stories = recursive_story_listdir(args.chapters_path) + log = args_obj.logger_with_filename() + + options = args.scan_types + to_remove = args.remove_option + + csv = OutputCSV(args.output_csv, options) + j = int(input("the index you left off at, if you are starting from scratch enter 0\n")) + + if to_remove is not None and j == 0: + shutil.copytree(args.chapters_path, args.chapters_backup_path) + rewrite = OutputStoryFiles(args.chapters_path) + + try: + # TODO: function that queries db for chapter and filenames + if j == 0: + csv.init_csv() + for s in stories[j:]: + log.info(f"story: {s}") + fname = os.path.join(args.chapters_path, s) + with open(fname, "r") as f: + text = f.readlines() + parser = Parser(text, options) + hits = parser.parse_lines() + + if to_remove is not None: + new_text = rewrite.strip_and_rewrite(text, hits) + rewrite.out_to_file(s, new_text) + + hits.update({'story_identifier':s}) + csv.write_data(hits) + j += 1 + + except KeyboardInterrupt: + print(f"progress at restart: {j}") + except Exception as e: + print(f"there's been an error: {e}\n") + print(f"progress at restart: {j}") From 62affd1cb70f42506f96161dfb90f5cc9ee8d2c8 Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Tue, 22 Sep 2026 12:50:39 -0500 Subject: [PATCH 2/5] updated mistake in README addition --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 536a052..9c54361 100755 --- a/README.md +++ b/README.md @@ -257,7 +257,7 @@ rows from the stories table in the final output database. ## Extract notes, summaries, and commercial links from chapters -Extracts notes, summaries, content warnings, and +Extracts notes, summaries, content warnings, and commercial links from story files This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. From bca989a2668c01973bb675c96847676ee8351ec2 Mon Sep 17 00:00:00 2001 From: thatgirlginger Date: Wed, 23 Sep 2026 10:26:32 -0500 Subject: [PATCH 3/5] ruff reformatting --- shared_python/Args.py | 10 +++--- shared_python/Common.py | 11 ++++-- story_cleanup/keywords.py | 52 ++++++++++++++++++++++++++--- story_cleanup/messages.py | 24 ++++++++++--- story_cleanup/outputs.py | 7 +++- story_cleanup/parsing.py | 7 ++-- xx-Extract-Notes-Summaries-Other.py | 16 +++++---- 7 files changed, 99 insertions(+), 28 deletions(-) diff --git a/shared_python/Args.py b/shared_python/Args.py index 59e12d1..4a7c1ba 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -282,13 +282,11 @@ def args_for_sn_extraction(self): ) if os.path.exists(self.args.chapters_backup_path): os.rmdir(self.args.chapters_backup_path) - + if self.args.output_csv is None: - self.args.output_csv = input( - "Output path for the extracted CSV" - ) + self.args.output_csv = input("Output path for the extracted CSV") if self.args.scan_types is None: - self.args.scan_types = ['notes', 'warnings', 'summary', 'commercial'] + self.args.scan_types = ["notes", "warnings", "summary", "commercial"] self._print_args(self.args) - return self.args \ No newline at end of file + return self.args diff --git a/shared_python/Common.py b/shared_python/Common.py index fb0afbd..c8b2509 100755 --- a/shared_python/Common.py +++ b/shared_python/Common.py @@ -15,11 +15,18 @@ def print_progress(cur, total, prog_type="stories"): sys.stdout.flush() return cur + def recursive_story_listdir(main_path): - storyfiles = [x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name] + storyfiles = [ + x.name for x in os.scandir(main_path) if "html" in x.name or "txt" in x.name + ] subdirs = [x.name for x in os.scandir(main_path) if x.is_dir()] for dir in subdirs: - stories = [x for x in os.listdir(os.path.join(main_path, dir)) if "html" in x or "txt" in x] + stories = [ + x + for x in os.listdir(os.path.join(main_path, dir)) + if "html" in x or "txt" in x + ] for s in stories: storyfiles.append(os.path.join(dir, s)) return storyfiles diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py index 27612eb..b274589 100644 --- a/story_cleanup/keywords.py +++ b/story_cleanup/keywords.py @@ -1,15 +1,57 @@ class Keywords: def __init__(self): - self.notes = ["note", "Note", "notes", "Notes", "A/n", "a/n", "A/N", "disclaimer", "Disclaimer", "DISCLAIMER"] - self.warnings = ["warning", "Warning", "warnings", "Warnings", "cw", "CW", "content warning", "Content Warning", "trigger warning", - "Trigger Warning", "Content warning", "Trigger warning", "tw", "TW"] - self.summary = ["summary", "Summary", "summaries", "Summaries", "prompt", "Prompt"] - self.commercial = ["Patreon", "patreon", "mailto", "Ko-fi", "ko-fi", "Ko-Fi", "gofundme", "GoFundMe"] + self.notes = [ + "note", + "Note", + "notes", + "Notes", + "A/n", + "a/n", + "A/N", + "disclaimer", + "Disclaimer", + "DISCLAIMER", + ] + self.warnings = [ + "warning", + "Warning", + "warnings", + "Warnings", + "cw", + "CW", + "content warning", + "Content Warning", + "trigger warning", + "Trigger Warning", + "Content warning", + "Trigger warning", + "tw", + "TW", + ] + self.summary = [ + "summary", + "Summary", + "summaries", + "Summaries", + "prompt", + "Prompt", + ] + self.commercial = [ + "Patreon", + "patreon", + "mailto", + "Ko-fi", + "ko-fi", + "Ko-Fi", + "gofundme", + "GoFundMe", + ] # self.ratings # self.wordcount # self.pairings # self.characters # others defined + def detect(self, type, text): match type: case "notes": diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py index 3bce060..bdf7e44 100644 --- a/story_cleanup/messages.py +++ b/story_cleanup/messages.py @@ -1,9 +1,11 @@ from colorama import Fore, Style + class Messaging: - ''' + """ class that handles interactivity, message colors, etc - ''' + """ + def __init__(self, text): self.text = text self.summary_color = Fore.BLUE @@ -13,6 +15,7 @@ def __init__(self, text): self.additional_lines_color = Fore.LIGHTBLACK_EX self.general = Fore.GREEN self.reset = Style.RESET_ALL + def check_init_response(self, type): match type: case "summary": @@ -23,14 +26,25 @@ def check_init_response(self, type): print(self.warning_color + f"{type}:\n" + self.reset + self.text) case "commercial": print(self.commercial_color + f"{type}:\n" + self.reset + self.text) - r = input(self.general + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + self.reset) + r = input( + self.general + + "Enter 'n' if this is not correct, otherwise press any key to continue: \n" + + self.reset + ) if r.lower() == "n": return False return True + def check_additional_lines(self, preview): print(f"{self.text}\n") print(self.additional_lines_color + preview + self.reset) - r = input((self.general + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + self.reset)) + r = input( + ( + self.general + + "Enter 'n' if you would like to add the next line to the string identified and removed, otherwise press any key to continue: \n" + + self.reset + ) + ) if r == "n": return True - return False \ No newline at end of file + return False diff --git a/story_cleanup/outputs.py b/story_cleanup/outputs.py index cf5a79a..42e75d8 100644 --- a/story_cleanup/outputs.py +++ b/story_cleanup/outputs.py @@ -5,19 +5,23 @@ class OutputCSV: def __init__(self, filename: str, columns: list): self.filename = filename - self.columns = columns + ['story_identifier'] + self.columns = columns + ["story_identifier"] + def init_csv(self): with open(self.filename, "w", encoding="utf_8_sig", newline="") as fp: csvf = csv.DictWriter(fp, fieldnames=self.columns) csvf.writeheader() + def write_data(self, data: dict): with open(self.filename, "a", encoding="utf_8_sig", newline="") as f: csvf = csv.DictWriter(f, fieldnames=self.columns) csvf.writerow(data) + class OutputStoryFiles: def __init__(self, output_path): self.output_path = output_path + def strip_and_rewrite(self, text, options: dict): text = "".join(text) for o in options.keys(): @@ -25,6 +29,7 @@ def strip_and_rewrite(self, text, options: dict): for h in hit_lines: text = text.replace(h, "") return text + def out_to_file(self, filename, strippedtext): with open(os.path.join(self.output_path, filename), "w") as f: f.write(strippedtext) diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py index 43ca135..56544c7 100644 --- a/story_cleanup/parsing.py +++ b/story_cleanup/parsing.py @@ -1,13 +1,14 @@ from .keywords import Keywords from .messages import Messaging + class Parser: def __init__(self, textlines: list, options: list): self.textlines = textlines self.max = len(self.textlines) self.options_dict = {} for opt in options: - self.options_dict.update({opt:""}) + self.options_dict.update({opt: ""}) self.end_message = "***no more lines to preview, press enter to continue***" self.kwords = Keywords() @@ -34,5 +35,7 @@ def parse_lines(self): for i in range(0, int(self.max)): for o in self.options_dict.keys(): if self.kwords.detect(o, self.textlines[i]): - self.options_dict[o] = self.options_dict[o] + "\n" + self.detection_loop(o, i) + self.options_dict[o] = ( + self.options_dict[o] + "\n" + self.detection_loop(o, i) + ) return self.options_dict diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py index 7497be0..171fb9b 100644 --- a/xx-Extract-Notes-Summaries-Other.py +++ b/xx-Extract-Notes-Summaries-Other.py @@ -8,11 +8,11 @@ from story_cleanup.parsing import Parser -#TODO: improve logging -''' +# TODO: improve logging +""" For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) -''' +""" if __name__ == "__main__": args_obj = Args() @@ -24,12 +24,14 @@ to_remove = args.remove_option csv = OutputCSV(args.output_csv, options) - j = int(input("the index you left off at, if you are starting from scratch enter 0\n")) + j = int( + input("the index you left off at, if you are starting from scratch enter 0\n") + ) if to_remove is not None and j == 0: shutil.copytree(args.chapters_path, args.chapters_backup_path) rewrite = OutputStoryFiles(args.chapters_path) - + try: # TODO: function that queries db for chapter and filenames if j == 0: @@ -45,8 +47,8 @@ if to_remove is not None: new_text = rewrite.strip_and_rewrite(text, hits) rewrite.out_to_file(s, new_text) - - hits.update({'story_identifier':s}) + + hits.update({"story_identifier": s}) csv.write_data(hits) j += 1 From fc246da906689edffc41de1a86dc412accc3ab89 Mon Sep 17 00:00:00 2001 From: gingermae Date: Wed, 14 Oct 2026 18:32:13 -0400 Subject: [PATCH 4/5] requested changes --- requirements.txt | 1 + story_cleanup/keywords.py | 32 ++++++-------------------------- 2 files changed, 7 insertions(+), 26 deletions(-) diff --git a/requirements.txt b/requirements.txt index 476d90d..d81daee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,3 +18,4 @@ pyyaml==6.0.1 colorlog==6.7.0 prompt_toolkit==3.0.39 ruff==0.1.5 +colorama==0.4.6 \ No newline at end of file diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py index b274589..8d6d1db 100644 --- a/story_cleanup/keywords.py +++ b/story_cleanup/keywords.py @@ -2,49 +2,28 @@ class Keywords: def __init__(self): self.notes = [ "note", - "Note", - "notes", - "Notes", - "A/n", "a/n", - "A/N", "disclaimer", - "Disclaimer", - "DISCLAIMER", + "tumblr", + "author" ] self.warnings = [ "warning", - "Warning", - "warnings", - "Warnings", "cw", - "CW", "content warning", - "Content Warning", "trigger warning", - "Trigger Warning", - "Content warning", - "Trigger warning", "tw", - "TW", ] self.summary = [ "summary", - "Summary", "summaries", - "Summaries", "prompt", - "Prompt", + "synopsis" ] self.commercial = [ - "Patreon", "patreon", - "mailto", - "Ko-fi", "ko-fi", - "Ko-Fi", - "gofundme", - "GoFundMe", + "kofundme", ] # self.ratings # self.wordcount @@ -67,6 +46,7 @@ def detect(self, type, text): "unsupported type: must be 'notes', 'warnings', 'summary', or 'commercial'; check config file" ) for k in keywords: - if k in text: + # eventually, a regex would be more efficient here, but with the line-by-line parsing built in this works for now + if k in text.lower(): return True return False From d56fe671238181463339fca8eeb6d3038ae0a1dc Mon Sep 17 00:00:00 2001 From: gingermae Date: Mon, 19 Oct 2026 18:11:03 -0400 Subject: [PATCH 5/5] cleaned up keywords list, created config options for additional files --- README.md | 2 + example.yml | 10 ++++- shared_python/Args.py | 13 +++++- story_cleanup/keywords.py | 63 +++++++++++++++++++---------- story_cleanup/messages.py | 4 +- story_cleanup/parsing.py | 4 +- xx-Extract-Notes-Summaries-Other.py | 10 +++-- 7 files changed, 75 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 9c54361..fd7f879 100755 --- a/README.md +++ b/README.md @@ -259,6 +259,8 @@ rows from the stories table in the final output database. Extracts notes, summaries, content warnings, and commercial links from story files +Due to the nature of fandom and archives, most are going to be different and have unique keywords. If your archive has additional keywords, you can enter them into a plaintext file for each category and add them to the .yml configuration file (see example.yml) + This is mostly for non-eFiction archives where the author's notes and content warnings are present in the main bodies of work instead of separate fields. If you use this with the 'remove_option' in the properties file populated, the script will create a backup of all works. If you hit an error or KeyboardInterrupt, the script prints out the index of the chapters list you were at. note that this does NOT load chapters into the stories table diff --git a/example.yml b/example.yml index 82758ef..a0e218b 100755 --- a/example.yml +++ b/example.yml @@ -37,4 +37,12 @@ remove_option: chapters_backup_path: /path/to/backup/chapter/story/output/files output_csv: /path/to/output/Desired_CSV_name.csv # Comma-separated list of types to scan for -scan_types: [notes, warnings, summary, commercial] +scan_types: [notes, warnings, summary, commercial, pairing, rating, wordcount] +additional_keywords: + - notes: path/to/additional_notes_keywords.txt + - warnings: path/to/additional_warnings_keywords.txt + - summary: path/to/additional_summary_keywords.txt + - commercial: + - pairing: + - rating: + - wordcount: \ No newline at end of file diff --git a/shared_python/Args.py b/shared_python/Args.py index 4a7c1ba..63e147b 100755 --- a/shared_python/Args.py +++ b/shared_python/Args.py @@ -286,7 +286,18 @@ def args_for_sn_extraction(self): if self.args.output_csv is None: self.args.output_csv = input("Output path for the extracted CSV") if self.args.scan_types is None: - self.args.scan_types = ["notes", "warnings", "summary", "commercial"] + self.args.scan_types = [ + "notes", + "warnings", + "summary", + "commercial", + "pairing", + "rating", + "wordcount", + ] + + if self.args.additional_keywords is not None: + self.args.additional_keywords = args.additional_keywords self._print_args(self.args) return self.args diff --git a/story_cleanup/keywords.py b/story_cleanup/keywords.py index 8d6d1db..b2202bf 100644 --- a/story_cleanup/keywords.py +++ b/story_cleanup/keywords.py @@ -1,12 +1,6 @@ class Keywords: - def __init__(self): - self.notes = [ - "note", - "a/n", - "disclaimer", - "tumblr", - "author" - ] + def __init__(self, additional_keywords={}): + self.notes = ["note", "a/n", "disclaimer", "tumblr", "author"] self.warnings = [ "warning", "cw", @@ -14,22 +8,44 @@ def __init__(self): "trigger warning", "tw", ] - self.summary = [ - "summary", - "summaries", - "prompt", - "synopsis" - ] + self.summary = ["summary", "summaries", "prompt", "synopsis"] self.commercial = [ "patreon", "ko-fi", "kofundme", ] - # self.ratings - # self.wordcount - # self.pairings - # self.characters - # others defined + self.rating = ["rating", "rated"] + self.wordcount = [ + "words", + "words", + "wordcount", + "word count", + ] + self.pairings = ["pairing", "relationship"] + + self.additional_keywords = additional_keywords + + def _open_file_and_read(self, keywordfile): + with open(keywordfile, "r") as f: + words = f.readlines() + return words + + def append_additional_keywords(self): + for t in self.additional_keywords.keys(): + words = self._open_file_and_read(self.additional_keywords[t]) + match t: + case "notes": + self.notes.append(words) + case "warnings": + self.warnings.append(words) + case "summary": + self.summary.append(words) + case "rating": + self.ratings.append(words) + case "wordcount": + self.wordcount.append(words) + case "pairings": + self.pairings.append(words) def detect(self, type, text): match type: @@ -41,12 +57,17 @@ def detect(self, type, text): keywords = self.summary case "commercial": keywords = self.commercial + case "rating": + keywords = self.ratings + case "wordcount": + keywords = self.wordcount + case "pairings": + keywords = self.pairings case _: raise NameError( - "unsupported type: must be 'notes', 'warnings', 'summary', or 'commercial'; check config file" + "unsupported type: must be 'notes', 'warnings', 'summary', 'rating', 'pairings', 'wordcount', or 'commercial'; check config file" ) for k in keywords: - # eventually, a regex would be more efficient here, but with the line-by-line parsing built in this works for now if k in text.lower(): return True return False diff --git a/story_cleanup/messages.py b/story_cleanup/messages.py index bdf7e44..6335957 100644 --- a/story_cleanup/messages.py +++ b/story_cleanup/messages.py @@ -20,10 +20,8 @@ def check_init_response(self, type): match type: case "summary": print(self.summary_color + f"{type}:\n" + self.reset + self.text) - case "notes": + case "notes" | "rating" | "wordcount" | "pairing" | "warnings": print(self.notes_color + f"{type}:\n" + self.reset + self.text) - case "warnings": - print(self.warning_color + f"{type}:\n" + self.reset + self.text) case "commercial": print(self.commercial_color + f"{type}:\n" + self.reset + self.text) r = input( diff --git a/story_cleanup/parsing.py b/story_cleanup/parsing.py index 56544c7..7663b11 100644 --- a/story_cleanup/parsing.py +++ b/story_cleanup/parsing.py @@ -3,14 +3,14 @@ class Parser: - def __init__(self, textlines: list, options: list): + def __init__(self, keywords, textlines: list, options: list): self.textlines = textlines self.max = len(self.textlines) self.options_dict = {} for opt in options: self.options_dict.update({opt: ""}) self.end_message = "***no more lines to preview, press enter to continue***" - self.kwords = Keywords() + self.keywords = keywords def detection_loop(self, type: str, i: int): m = Messaging(self.textlines[i]) diff --git a/xx-Extract-Notes-Summaries-Other.py b/xx-Extract-Notes-Summaries-Other.py index 171fb9b..36131d5 100644 --- a/xx-Extract-Notes-Summaries-Other.py +++ b/xx-Extract-Notes-Summaries-Other.py @@ -7,8 +7,6 @@ from story_cleanup.outputs import OutputCSV, OutputStoryFiles from story_cleanup.parsing import Parser - -# TODO: improve logging """ For archives where authors' notes and summaries are in the main body of text, this script scans for and removes them note that this does /not/ load fields into the database tables, only extracts them to a CSV (and deletes if necessary) @@ -22,6 +20,7 @@ options = args.scan_types to_remove = args.remove_option + additional_keywords = args.additional_keywords csv = OutputCSV(args.output_csv, options) j = int( @@ -32,6 +31,11 @@ shutil.copytree(args.chapters_path, args.chapters_backup_path) rewrite = OutputStoryFiles(args.chapters_path) + if additional_keywords: + keywords = Keywords(additional_keywords) + else: + keywords = Keywords() + try: # TODO: function that queries db for chapter and filenames if j == 0: @@ -41,7 +45,7 @@ fname = os.path.join(args.chapters_path, s) with open(fname, "r") as f: text = f.readlines() - parser = Parser(text, options) + parser = Parser(keywords, text, options) hits = parser.parse_lines() if to_remove is not None: