From 22ee146b1be4a5c60a05de4b61f826e90cac8b01 Mon Sep 17 00:00:00 2001 From: jowj Date: Thu, 6 Feb 2020 13:55:20 -0600 Subject: [PATCH] Usability improvements: - refactor arguments to parse args instead environment vars - move to single file instead of two files - add pdb.pm() functionality for debugging after a crash --- pinboard.py | 62 -------------------- pynit.py | 160 ++++++++++++++++++++++++++++++++++++++++++++++++++++ reddit.py | 58 ------------------- 3 files changed, 160 insertions(+), 120 deletions(-) delete mode 100644 pinboard.py create mode 100644 pynit.py delete mode 100644 reddit.py diff --git a/pinboard.py b/pinboard.py deleted file mode 100644 index 2b2046a..0000000 --- a/pinboard.py +++ /dev/null @@ -1,62 +0,0 @@ -import json -import os -import time -import requests - - -PINBOARD_TOKEN = os.environ.get("PINBOARD_TOKEN") -PINBOARD_BASE_URL = "https://api.pinboard.in/v1/" -PINBOARD_AUTH_SNIPPET = f"?auth_token={PINBOARD_TOKEN}" - -def get_all_posts(): - """ - returns a list of all pins in pinboard account - """ - get_post_snippet = f"posts/all?auth_token={PINBOARD_TOKEN}" - pinboard_url = PINBOARD_BASE_URL + get_post_snippet - - return requests.get(pinboard_url) - - -def add_pin_url(reddit_dict): - """ - adds a pin to pinboard and returns the response - """ - add_post_snippet = "posts/add" - # headers = {'Content-type': 'application/json'} - args = { - 'url': reddit_dict['url'], - 'description': reddit_dict['title'], - 'extended': reddit_dict['description'], - 'tags': reddit_dict['tag'], - 'replace': 'no' - } - - post_url = PINBOARD_BASE_URL + add_post_snippet + PINBOARD_AUTH_SNIPPET - - response = requests.get(post_url, params=args) - # pdb.set_trace() - print(response.text) - return response - - -def import_reddit_url_from_file(filename): - """ - imports a list of reddit URLs and meta data from a file. - returns a json object of that data. - """ - with open(filename, 'r') as infile: - data = json.loads(infile.read()) - - return data - - -if __name__ == "__main__": - """ - You have to sleep for 3 seconds between requests or Maciej will Get Unhappy - per https://pinboard.in/api - """ - REDDIT_DATA = import_reddit_url_from_file("data.json") - for entry in REDDIT_DATA: - post_response = add_pin_url(entry) - time.sleep(3) diff --git a/pynit.py b/pynit.py new file mode 100644 index 0000000..0588c9d --- /dev/null +++ b/pynit.py @@ -0,0 +1,160 @@ +import argparse +import logging +import sys +import time +import requests + +import json +import praw + +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) + + +def munge_idiot_data(reddit_dict): + """ + this function handles converting reddit relative urls to fully qualified urls. + its extremely fucking unclear which *.url properties will give you fully qualified urls, + so rather than handlign this properly by just fixing the broken ones, i'm going to inspect + every url that comes through my apparatus. + """ + protocol = 'https' + # pdb.set_trace() + for single_dict in reddit_dict: + if protocol in single_dict['url']: + pass + else: + single_dict['url'] = 'https://reddit.com' + single_dict['url'] + + return reddit_dict + + + +def get_all_posts(token, base_url): + """ + returns a list of all pins in pinboard account + """ + get_post_snippet = f"posts/all?auth_token={token}" + pinboard_url = base_url + get_post_snippet + + return requests.get(pinboard_url) + + +def add_pin_url(reddit_dict, base_url, auth_snippet): + """ + adds a pin to pinboard and returns the response + """ + add_post_snippet = "posts/add" + # headers = {'Content-type': 'application/json'} + args = { + 'url': reddit_dict['url'], + 'description': reddit_dict['title'], + 'extended': reddit_dict['description'], + 'tags': reddit_dict['tag'], + 'replace': 'no' + } + + post_url = base_url + add_post_snippet + auth_snippet + response = requests.get(post_url, params=args) + + return response + + +def import_reddit_url_from_file(filename): + """ + imports a list of reddit URLs and meta data from a file. + returns a json object of that data. + """ + with open(filename, 'r') as infile: + data = json.loads(infile.read()) + + return data + + +def idb_excepthook(type, value, tb): + """Call an interactive debugger in post-mortem mode + If you do "sys.excepthook = idb_excepthook", then an interactive debugger + will be spawned at an unhandled exception + """ + if hasattr(sys, 'ps1') or not sys.stderr.isatty(): + sys.__excepthook__(type, value, tb) + else: + import pdb, traceback + traceback.print_exception(type, value, tb) + print + pdb.pm() + + +def main(*args, **kwargs): + parser = argparse.ArgumentParser( + description="pynit: an integration between saved reddit posts and pinboard.") + parser.add_argument( + "--debug", "-d", action='store_true', + help="Include debugging output") + parser.add_argument( + "--reddit-un", "-run", required=True, help="Reddit username") + parser.add_argument( + "--reddit-pw", "-rpw", required=True, help="Reddit password") + parser.add_argument( + "--reddit-cid", "-rcid", required=True, help="Reddit client id") + parser.add_argument( + "--reddit-sec", "-rsec", required=True, help="Reddit client secret") + parser.add_argument( + "--pb-apikey", "-pba", required=True, help="Pinboard api key") + parsed = parser.parse_args() + if parsed.debug: + sys.excepthook = idb_excepthook + LOGGER.setLevel(logging.DEBUG) + + reddit = praw.Reddit(client_id=parsed.reddit_cid, + client_secret=parsed.reddit_sec, + user_agent='/u/ pynit-tasks', + username=parsed.reddit_un, + password=parsed.reddit_pw + ) + + # this line is the most cursed line in programming + # REDDIT.redditor, + your_user = reddit.redditor(parsed.reddit_un) + saved_posts = your_user.saved(limit=None) + + POSTS_TO_SAVE = [] + for link in saved_posts: + if hasattr(link, 'is_self'): + POSTS_TO_SAVE.append({ + 'title': link.title, + 'tag': link.subreddit.display_name + ' added-by-pynnit', + 'description': link.selftext, + 'url': link.permalink + }) + elif hasattr(link, 'is_root'): + POSTS_TO_SAVE.append({ + 'title': link.link_title, + 'tag': link.subreddit.display_name + ' added-by-pynnit', + 'description': link.body, + 'url': link.link_url + }) + else: + print("shit is fucked.") + + MUNGED_DATA = munge_idiot_data(POSTS_TO_SAVE) + with open('data.json', 'w') as outfile: + json.dump(MUNGED_DATA, outfile, indent=2) + + # handle the pinboard side of things + pinboard_token = parsed.pb_apikey + pinboard_base_url = "https://api.pinboard.in/v1/" + pinboard_auth_snippet = f"?auth_token={pinboard_token}" + + """ + You have to sleep for 3 seconds between requests or Maciej will Get Unhappy + per https://pinboard.in/api + """ + reddit_data = import_reddit_url_from_file("data.json") + for entry in reddit_data: + post_response = add_pin_url(entry, pinboard_base_url, pinboard_auth_snippet) + time.sleep(3) + + +if __name__ == '__main__': + sys.exit(main(*sys.argv)) diff --git a/reddit.py b/reddit.py deleted file mode 100644 index cce544f..0000000 --- a/reddit.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -import os -import praw - - -def munge_idiot_data(reddit_dict): - """ - this function handles converting reddit relative urls to fully qualified urls. - its extremely fucking unclear which *.url properties will give you fully qualified urls, - so rather than handlign this properly by just fixing the broken ones, i'm going to inspect - every url that comes through my apparatus. - """ - protocol = 'https' - # pdb.set_trace() - for single_dict in reddit_dict: - if protocol in single_dict['url']: - pass - else: - single_dict['url'] = 'https://reddit.com' + single_dict['url'] - - return reddit_dict - - -if __name__ == "__main__": - REDDIT = praw.Reddit(client_id=os.environ.get('REDDIT_ID'), - client_secret=os.environ.get('REDDIT_SECRET'), - user_agent='/u/ pynit-tasks', - username=os.environ.get('REDDIT_UN'), - password=os.environ.get('REDDIT_PW') - ) - - # this line is the most cursed line in programming - # REDDIT.redditor, - YOUR_USER = REDDIT.redditor(os.environ.get('REDDIT_UN')) - SAVED_POSTS = YOUR_USER.saved(limit=None) - - POSTS_TO_SAVE = [] - for link in SAVED_POSTS: - if hasattr(link, 'is_self'): - POSTS_TO_SAVE.append({ - 'title': link.title, - 'tag': link.subreddit.display_name + ' added-by-pynnit', - 'description': link.selftext, - 'url': link.permalink - }) - elif hasattr(link, 'is_root'): - POSTS_TO_SAVE.append({ - 'title': link.link_title, - 'tag': link.subreddit.display_name + ' added-by-pynnit', - 'description': link.body, - 'url': link.link_url - }) - else: - print("shit is fucked.") - - MUNGED_DATA = munge_idiot_data(POSTS_TO_SAVE) - with open('data.json', 'w') as outfile: - json.dump(MUNGED_DATA, outfile, indent=2)