Compare commits
4 Commits
a62d316569
...
333d9f7486
Author | SHA1 | Date | |
---|---|---|---|
333d9f7486 | |||
31922b1a02 | |||
86bf6c42e3 | |||
22ee146b1b |
27
README.md
27
README.md
@ -26,27 +26,14 @@ https://github.com/reddit-archive/reddit/wiki/oauth2
|
||||
Your pinboard api key can be found here (assuming you are logged in):
|
||||
https://pinboard.in/settings/password/
|
||||
|
||||
You need to export these in your shell in the following format:
|
||||
```
|
||||
export REDDIT_UN=''
|
||||
export REDDIT_PW=''
|
||||
export REDDIT_ID=''
|
||||
export REDDIT_SECRET=''
|
||||
export PINBOARD_TOKEN=''
|
||||
```
|
||||
|
||||
Then run the following commands, in the order given:
|
||||
|
||||
```
|
||||
python main.py
|
||||
python pinboard.py
|
||||
```
|
||||
You then pass those variables as arguments to the script run:
|
||||
`☭ python pynit.py --reddit-un '' --reddit-pw '' --reddit-cid '' --reddit-sec '' --pb-apikey ''`
|
||||
|
||||
## background and additional information
|
||||
### getting data from reddit
|
||||
if you don't have MFA set up for your account (you should do that) then this is easy. everything works as expected. BUT if you DO have MFA set up let me tell you: this gets dumber. when you export your reddit password you have to also include an active MFA 6 digit code, like this:
|
||||
if you don't have MFA set up for your account (you should do that) then this is easy. everything works as expected. BUT if you DO have MFA set up let me tell you: this gets dumber. when you enter your reddit password you have to also include an active MFA 6 digit code, like this:
|
||||
|
||||
`export REDDIT_PW='password:123456'`
|
||||
`... --redditpw "thing:123123"`
|
||||
|
||||
so good.
|
||||
### putting data in pinboard
|
||||
@ -61,6 +48,6 @@ Reddit /comments/ do not have this attribute. They DO have an attribute called `
|
||||
|
||||
# roadmap
|
||||
|
||||
- [ ] move reddit/pinboard script to single file.
|
||||
- [ ] make the script take command line arguments (i.e. must pass `-reddit` if you want to sync saved posts)
|
||||
- [ ] write new function `update_tags` that will take a tag and retag to a new name (add a cli flag)
|
||||
- [X] move reddit/pinboard script to single file.
|
||||
- [x] make the script take command line arguments (i.e. must pass `-reddit` if you want to sync saved posts)
|
||||
- [x] write new function `update_tags` that will take a tag and retag to a new name (add a cli flag)
|
||||
|
62
pinboard.py
62
pinboard.py
@ -1,62 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import requests
|
||||
|
||||
|
||||
PINBOARD_TOKEN = os.environ.get("PINBOARD_TOKEN")
|
||||
PINBOARD_BASE_URL = "https://api.pinboard.in/v1/"
|
||||
PINBOARD_AUTH_SNIPPET = f"?auth_token={PINBOARD_TOKEN}"
|
||||
|
||||
def get_all_posts():
|
||||
"""
|
||||
returns a list of all pins in pinboard account
|
||||
"""
|
||||
get_post_snippet = f"posts/all?auth_token={PINBOARD_TOKEN}"
|
||||
pinboard_url = PINBOARD_BASE_URL + get_post_snippet
|
||||
|
||||
return requests.get(pinboard_url)
|
||||
|
||||
|
||||
def add_pin_url(reddit_dict):
|
||||
"""
|
||||
adds a pin to pinboard and returns the response
|
||||
"""
|
||||
add_post_snippet = "posts/add"
|
||||
# headers = {'Content-type': 'application/json'}
|
||||
args = {
|
||||
'url': reddit_dict['url'],
|
||||
'description': reddit_dict['title'],
|
||||
'extended': reddit_dict['description'],
|
||||
'tags': reddit_dict['tag'],
|
||||
'replace': 'no'
|
||||
}
|
||||
|
||||
post_url = PINBOARD_BASE_URL + add_post_snippet + PINBOARD_AUTH_SNIPPET
|
||||
|
||||
response = requests.get(post_url, params=args)
|
||||
# pdb.set_trace()
|
||||
print(response.text)
|
||||
return response
|
||||
|
||||
|
||||
def import_reddit_url_from_file(filename):
|
||||
"""
|
||||
imports a list of reddit URLs and meta data from a file.
|
||||
returns a json object of that data.
|
||||
"""
|
||||
with open(filename, 'r') as infile:
|
||||
data = json.loads(infile.read())
|
||||
|
||||
return data
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
You have to sleep for 3 seconds between requests or Maciej will Get Unhappy
|
||||
per https://pinboard.in/api
|
||||
"""
|
||||
REDDIT_DATA = import_reddit_url_from_file("data.json")
|
||||
for entry in REDDIT_DATA:
|
||||
post_response = add_pin_url(entry)
|
||||
time.sleep(3)
|
187
pynit.py
Normal file
187
pynit.py
Normal file
@ -0,0 +1,187 @@
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import requests
|
||||
|
||||
import json
|
||||
import praw
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def munge_idiot_data(reddit_dict):
|
||||
"""
|
||||
this function handles converting reddit relative urls to fully qualified urls.
|
||||
its extremely fucking unclear which *.url properties will give you fully qualified urls,
|
||||
so rather than handlign this properly by just fixing the broken ones, i'm going to inspect
|
||||
every url that comes through my apparatus.
|
||||
"""
|
||||
protocol = 'https'
|
||||
# pdb.set_trace()
|
||||
for single_dict in reddit_dict:
|
||||
if protocol in single_dict['url']:
|
||||
pass
|
||||
else:
|
||||
single_dict['url'] = 'https://reddit.com' + single_dict['url']
|
||||
|
||||
return reddit_dict
|
||||
|
||||
|
||||
|
||||
def get_all_posts(token, base_url):
|
||||
"""
|
||||
returns a list of all pins in pinboard account
|
||||
"""
|
||||
get_post_snippet = f"posts/all?auth_token={token}"
|
||||
pinboard_url = base_url + get_post_snippet
|
||||
|
||||
return requests.get(pinboard_url)
|
||||
|
||||
|
||||
def add_pin_url(reddit_dict, base_url, auth_snippet):
|
||||
"""
|
||||
adds a pin to pinboard and returns the response
|
||||
"""
|
||||
add_post_snippet = "posts/add"
|
||||
# headers = {'Content-type': 'application/json'}
|
||||
args = {
|
||||
'url': reddit_dict['url'],
|
||||
'description': reddit_dict['title'],
|
||||
'extended': reddit_dict['description'],
|
||||
'tags': reddit_dict['tag'],
|
||||
'replace': 'no'
|
||||
}
|
||||
|
||||
post_url = base_url + add_post_snippet + auth_snippet
|
||||
response = requests.get(post_url, params=args)
|
||||
|
||||
return response
|
||||
|
||||
def update_pin_tag(old_tag, new_tag, base_url, auth_snippet):
|
||||
rename_tags_snippet = "tags/rename"
|
||||
# headers = {'Content-type': 'application/json'}
|
||||
args = {
|
||||
'old': old_tag,
|
||||
'new': new_tag
|
||||
}
|
||||
|
||||
post_url = base_url + rename_tags_snippet + auth_snippet
|
||||
response = requests.get(post_url, params=args)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
|
||||
def import_reddit_url_from_file(filename):
|
||||
"""
|
||||
imports a list of reddit URLs and meta data from a file.
|
||||
returns a json object of that data.
|
||||
"""
|
||||
with open(filename, 'r') as infile:
|
||||
data = json.loads(infile.read())
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def idb_excepthook(type, value, tb):
|
||||
"""Call an interactive debugger in post-mortem mode
|
||||
If you do "sys.excepthook = idb_excepthook", then an interactive debugger
|
||||
will be spawned at an unhandled exception
|
||||
"""
|
||||
if hasattr(sys, 'ps1') or not sys.stderr.isatty():
|
||||
sys.__excepthook__(type, value, tb)
|
||||
else:
|
||||
import pdb, traceback
|
||||
traceback.print_exception(type, value, tb)
|
||||
print
|
||||
pdb.pm()
|
||||
|
||||
|
||||
def main(*args, **kwargs):
|
||||
parser = argparse.ArgumentParser(
|
||||
description="pynit: an integration between saved reddit posts and pinboard.")
|
||||
parser.add_argument(
|
||||
"--debug", "-d", action='store_true', help="Include debugging output")
|
||||
parser.add_argument(
|
||||
"--retag", "-rt", action='store_true',
|
||||
help="Select this if you want to update an existing tag")
|
||||
parser.add_argument(
|
||||
"--reddit-un", "-run", required=True, help="Reddit username")
|
||||
parser.add_argument(
|
||||
"--reddit-pw", "-rpw", required=True, help="Reddit password")
|
||||
parser.add_argument(
|
||||
"--reddit-cid", "-rcid", required=True, help="Reddit client id")
|
||||
parser.add_argument(
|
||||
"--reddit-sec", "-rsec", required=True, help="Reddit client secret")
|
||||
parser.add_argument(
|
||||
"--pb-apikey", "-pba", required=True, help="Pinboard api key")
|
||||
parsed = parser.parse_args()
|
||||
if parsed.debug:
|
||||
sys.excepthook = idb_excepthook
|
||||
LOGGER.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
pinboard_token = parsed.pb_apikey
|
||||
pinboard_base_url = "https://api.pinboard.in/v1/"
|
||||
pinboard_auth_snippet = f"?auth_token={pinboard_token}"
|
||||
|
||||
if parsed.retag:
|
||||
original_tag = input('what tag would you like to replace?')
|
||||
updated_tag = input('what tag would you like to use instead?')
|
||||
|
||||
update_pin_tag(original_tag, updated_tag, pinboard_base_url, pinboard_auth_snippet)
|
||||
|
||||
return
|
||||
|
||||
reddit = praw.Reddit(client_id=parsed.reddit_cid,
|
||||
client_secret=parsed.reddit_sec,
|
||||
user_agent='/u/ pynit-tasks',
|
||||
username=parsed.reddit_un,
|
||||
password=parsed.reddit_pw
|
||||
)
|
||||
|
||||
# this line is the most cursed line in programming
|
||||
# REDDIT.redditor,
|
||||
your_user = reddit.redditor(parsed.reddit_un)
|
||||
saved_posts = your_user.saved(limit=None)
|
||||
|
||||
POSTS_TO_SAVE = []
|
||||
for link in saved_posts:
|
||||
if hasattr(link, 'is_self'):
|
||||
POSTS_TO_SAVE.append({
|
||||
'title': link.title,
|
||||
'tag': link.subreddit.display_name + ' added-by-pynnit',
|
||||
'description': link.selftext,
|
||||
'url': link.permalink
|
||||
})
|
||||
elif hasattr(link, 'is_root'):
|
||||
POSTS_TO_SAVE.append({
|
||||
'title': link.link_title,
|
||||
'tag': link.subreddit.display_name + ' added-by-pynnit',
|
||||
'description': link.body,
|
||||
'url': link.link_url
|
||||
})
|
||||
else:
|
||||
print("shit is fucked.")
|
||||
|
||||
MUNGED_DATA = munge_idiot_data(POSTS_TO_SAVE)
|
||||
with open('data.json', 'w') as outfile:
|
||||
json.dump(MUNGED_DATA, outfile, indent=2)
|
||||
|
||||
# handle the pinboard side of things
|
||||
|
||||
|
||||
"""
|
||||
You have to sleep for 3 seconds between requests or Maciej will Get Unhappy
|
||||
per https://pinboard.in/api
|
||||
"""
|
||||
reddit_data = import_reddit_url_from_file("data.json")
|
||||
for entry in reddit_data:
|
||||
post_response = add_pin_url(entry, pinboard_base_url, pinboard_auth_snippet)
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main(*sys.argv))
|
58
reddit.py
58
reddit.py
@ -1,58 +0,0 @@
|
||||
import json
|
||||
import os
|
||||
import praw
|
||||
|
||||
|
||||
def munge_idiot_data(reddit_dict):
|
||||
"""
|
||||
this function handles converting reddit relative urls to fully qualified urls.
|
||||
its extremely fucking unclear which *.url properties will give you fully qualified urls,
|
||||
so rather than handlign this properly by just fixing the broken ones, i'm going to inspect
|
||||
every url that comes through my apparatus.
|
||||
"""
|
||||
protocol = 'https'
|
||||
# pdb.set_trace()
|
||||
for single_dict in reddit_dict:
|
||||
if protocol in single_dict['url']:
|
||||
pass
|
||||
else:
|
||||
single_dict['url'] = 'https://reddit.com' + single_dict['url']
|
||||
|
||||
return reddit_dict
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
REDDIT = praw.Reddit(client_id=os.environ.get('REDDIT_ID'),
|
||||
client_secret=os.environ.get('REDDIT_SECRET'),
|
||||
user_agent='/u/ pynit-tasks',
|
||||
username=os.environ.get('REDDIT_UN'),
|
||||
password=os.environ.get('REDDIT_PW')
|
||||
)
|
||||
|
||||
# this line is the most cursed line in programming
|
||||
# REDDIT.redditor,
|
||||
YOUR_USER = REDDIT.redditor(os.environ.get('REDDIT_UN'))
|
||||
SAVED_POSTS = YOUR_USER.saved(limit=None)
|
||||
|
||||
POSTS_TO_SAVE = []
|
||||
for link in SAVED_POSTS:
|
||||
if hasattr(link, 'is_self'):
|
||||
POSTS_TO_SAVE.append({
|
||||
'title': link.title,
|
||||
'tag': link.subreddit.display_name + ' added-by-pynnit',
|
||||
'description': link.selftext,
|
||||
'url': link.permalink
|
||||
})
|
||||
elif hasattr(link, 'is_root'):
|
||||
POSTS_TO_SAVE.append({
|
||||
'title': link.link_title,
|
||||
'tag': link.subreddit.display_name + ' added-by-pynnit',
|
||||
'description': link.body,
|
||||
'url': link.link_url
|
||||
})
|
||||
else:
|
||||
print("shit is fucked.")
|
||||
|
||||
MUNGED_DATA = munge_idiot_data(POSTS_TO_SAVE)
|
||||
with open('data.json', 'w') as outfile:
|
||||
json.dump(MUNGED_DATA, outfile, indent=2)
|
@ -1,5 +0,0 @@
|
||||
export REDDIT_UN=''
|
||||
export REDDIT_PW=''
|
||||
export REDDIT_ID=''
|
||||
export REDDIT_SECRET=''
|
||||
export PINBOARD_TOKEN=''
|
Loading…
Reference in New Issue
Block a user