Compare commits

...

5 Commits

Author SHA1 Message Date
josiah 0515d6d9db Update docs to reflect poetry instructions.
4 years ago
josiah 63a5afc50b Fix typoes and markdown blunders; update roadmap.
4 years ago
josiah cb61ff6d6c Fix some empty lines, remove comments, lowercase non-globals.
4 years ago
josiah 24a27c0776 Move from argparse required args to individually handled args.
4 years ago
josiah 0f9e3c61d3 Move from pipenv to poetry, since that's what the Cool Kids do now.
4 years ago

@ -1,21 +0,0 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
[packages]
requests = "==2.22.0"
praw = "==6.4.0"
certifi = "==2019.9.11"
chardet = "==3.0.4"
idna = "==2.8"
prawcore = "==1.0.1"
six = "==1.13.0"
urllib3 = "==1.25.6"
update_checker = "==0.16"
websocket_client = "==0.56.0"
[requires]
python_version = "3.7"

101
Pipfile.lock generated

@ -1,101 +0,0 @@
{
"_meta": {
"hash": {
"sha256": "6b64c77ab5a55c11d34da51420b7da13e4ffe1038bdf239f3a06a52dfe35d1bc"
},
"pipfile-spec": 6,
"requires": {
"python_version": "3.7"
},
"sources": [
{
"name": "pypi",
"url": "https://pypi.org/simple",
"verify_ssl": true
}
]
},
"default": {
"certifi": {
"hashes": [
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
],
"index": "pypi",
"version": "==2019.9.11"
},
"chardet": {
"hashes": [
"sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
"sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
],
"index": "pypi",
"version": "==3.0.4"
},
"idna": {
"hashes": [
"sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407",
"sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c"
],
"index": "pypi",
"version": "==2.8"
},
"praw": {
"hashes": [
"sha256:2e5c98e49fe60e5308255ed147b670d350f98281f84f582df30f87de727b6de2",
"sha256:cb8f85541ad4c6b10214ef9639acccfb5fed7ffee977be169b85357d2d2ea6d9"
],
"index": "pypi",
"version": "==6.4.0"
},
"prawcore": {
"hashes": [
"sha256:25dd14bf121bc0ad2ffc78e2322d9a01a516017105a5596cc21bb1e9a928b40c",
"sha256:ab5558efb438aa73fc66c4178bfc809194dea3ce2addf4dec873de7e2fd2824e"
],
"index": "pypi",
"version": "==1.0.1"
},
"requests": {
"hashes": [
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
],
"index": "pypi",
"version": "==2.22.0"
},
"six": {
"hashes": [
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
],
"index": "pypi",
"version": "==1.13.0"
},
"update-checker": {
"hashes": [
"sha256:59cfad7f9a0ee99f95f1dfc60f55bf184937bcab46a7270341c2c33695572453",
"sha256:70e39446fccf77b21192cf7a8214051fa93a636dc3b5c8b602b589d100a168b8"
],
"index": "pypi",
"version": "==0.16"
},
"urllib3": {
"hashes": [
"sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
"sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
],
"index": "pypi",
"version": "==1.25.6"
},
"websocket-client": {
"hashes": [
"sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9",
"sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a"
],
"index": "pypi",
"version": "==0.56.0"
}
},
"develop": {}
}

@ -10,7 +10,7 @@ specifically, comments and posts saved to your reddit user profile can be pulled
the tag "added-by-pynnit" is also added to each entry moved to pinboard in this way. This allows for easy viewing of all imported links. I found this very useful when I was writing the script in the first place, but you may not want it.
## using this bullshit
### installing
I use `pipenv` pretty exclusively because of how it works with my editor. that's the only package manager i've actually tested with, but theoretically the typical `pip install -r requirements.txt` should work just fine.
I use poetry a lot. ~poetry install~ should do you just fine.
### setting up the environment
You need several things to use this project:
@ -26,7 +26,7 @@ https://github.com/reddit-archive/reddit/wiki/oauth2
Your pinboard api key can be found here (assuming you are logged in):
https://pinboard.in/settings/password/
You then pass those variables as arguments to the script run:
You then pass those variables as arguments to the script when run:
### syncing posts from reddit
@ -35,7 +35,7 @@ You then pass those variables as arguments to the script run:
fill in your information and off you go!
### updating tags
To be specfici, this allows you to update tags _in bulk_; if you have 80 items under the 'clothing' tag, and 40 items under the 'clothes' tag you can combine them by running this code
To be specific, this allows you to update tags _in bulk_; if you have 80 items under the 'clothing' tag, and 40 items under the 'clothes' tag you can combine them by running this code
`☭ python pynit.py -rt --reddit-un '' --reddit-pw '' --reddit-cid '' --reddit-sec '' --pb-apikey ''`
You will then be asked for the tag you want to replace, and what you want to replace it with. This is an entirely `pinboard` side operation; there's no technical reason that the reddit information should be required, I just haven't split that off the main loop yet. As long as the flags are present & the `pb-apikey` is correct you will be able to update tags. You will not need _accurate_ reddit creds for this operation. This will eventually be fixed!
@ -43,7 +43,7 @@ You will then be asked for the tag you want to replace, and what you want to rep
### running in debug mode
`☭ python pynit.py -d --reddit-un '' --reddit-pw '' --reddit-cid '' --reddit-sec '' --pb-apikey ''`
Thanks to @mrled for this. I can't believe there's still so much I don't know about `pdb`!! The `-d` flag will do nothing unless the script encounters an exception. If it does, it'll dump you into the `pdb.pm()` dbg repl. Its SUPER useful while you're tweaking stuff / catching idiot mistakes with your password/mfa/whatever.
Thanks to [@micah](https://me.micahrl.com/) for this. I can't believe there's still so much I don't know about `pdb`!! The `-d` flag will do nothing unless the script encounters an exception. If it does, it'll dump you into the `pdb.pm()` dbg repl. Its SUPER useful while you're tweaking stuff / catching idiot mistakes with your password/mfa/whatever.
## background and additional information
### getting data from reddit
@ -67,4 +67,5 @@ Reddit /comments/ do not have this attribute. They DO have an attribute called `
- [x] move reddit/pinboard script to single file.
- [x] make the script take command line arguments (i.e. must pass `-reddit` if you want to sync saved posts)
- [x] write new function `update_tags` that will take a tag and retag to a new name (add a cli flag)
- [ ] Remove requirement for unnecessary creds when doing `pinboard` specific operations
- [x] remove requirement for unnecessary creds when doing `pinboard` specific operations
- [x] move to poetry from pipenv

167
poetry.lock generated

@ -0,0 +1,167 @@
[[package]]
category = "main"
description = "Python package for providing Mozilla's CA Bundle."
name = "certifi"
optional = false
python-versions = "*"
version = "2020.4.5.1"
[[package]]
category = "main"
description = "Universal encoding detector for Python 2 and 3"
name = "chardet"
optional = false
python-versions = "*"
version = "3.0.4"
[[package]]
category = "main"
description = "Internationalized Domain Names in Applications (IDNA)"
name = "idna"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.9"
[[package]]
category = "main"
description = "PRAW, an acronym for `Python Reddit API Wrapper`, is a python package that allows for simple access to reddit's API."
name = "praw"
optional = false
python-versions = ">=3.5"
version = "7.0.0"
[package.dependencies]
prawcore = ">=1.3.0,<2.0"
update-checker = ">=0.16"
websocket-client = ">=0.54.0"
[package.extras]
ci = ["coveralls"]
dev = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "black", "flake8", "pre-commit", "pydocstyle", "pytest (>=2.7.3)", "sphinx (<3.0)", "sphinx-rtd-theme"]
lint = ["black", "flake8", "pydocstyle", "sphinx (<3.0)", "sphinx-rtd-theme"]
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.3.0,<0.5)", "pytest (>=2.7.3)"]
[[package]]
category = "main"
description = "Low-level communication layer for PRAW 4+."
name = "prawcore"
optional = false
python-versions = "*"
version = "1.3.0"
[package.dependencies]
requests = ">=2.6.0,<3.0"
[package.extras]
ci = ["coveralls"]
dev = ["black", "flake8", "pre-commit", "pydocstyle", "betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
lint = ["black", "flake8", "pre-commit", "pydocstyle"]
test = ["betamax (>=0.8,<0.9)", "betamax-matchers (>=0.4.0,<0.5)", "betamax-serializers (>=0.2.0,<0.3)", "mock (>=0.8)", "pytest", "testfixtures (>4.13.2,<7)"]
[[package]]
category = "main"
description = "Python HTTP for Humans."
name = "requests"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
version = "2.23.0"
[package.dependencies]
certifi = ">=2017.4.17"
chardet = ">=3.0.2,<4"
idna = ">=2.5,<3"
urllib3 = ">=1.21.1,<1.25.0 || >1.25.0,<1.25.1 || >1.25.1,<1.26"
[package.extras]
security = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7)", "win-inet-pton"]
[[package]]
category = "main"
description = "Python 2 and 3 compatibility utilities"
name = "six"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
version = "1.15.0"
[[package]]
category = "main"
description = "A python module that will check for package updates."
name = "update-checker"
optional = false
python-versions = "*"
version = "0.17"
[package.dependencies]
requests = ">=2.3.0"
[[package]]
category = "main"
description = "HTTP library with thread-safe connection pooling, file post, and more."
name = "urllib3"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4"
version = "1.25.9"
[package.extras]
brotli = ["brotlipy (>=0.6.0)"]
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "pyOpenSSL (>=0.14)", "ipaddress"]
socks = ["PySocks (>=1.5.6,<1.5.7 || >1.5.7,<2.0)"]
[[package]]
category = "main"
description = "WebSocket client for Python. hybi13 is supported."
name = "websocket-client"
optional = false
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.57.0"
[package.dependencies]
six = "*"
[metadata]
content-hash = "1259b143f3221b52c498d529bbb3818aa639fc3ddaa6bb7c8f39777260433048"
python-versions = "^3.7"
[metadata.files]
certifi = [
{file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"},
{file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
]
idna = [
{file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"},
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
]
praw = [
{file = "praw-7.0.0-py2-none-any.whl", hash = "sha256:dcdcf13b7f7ae2393afd914644bf16b254eaf5230c81adf2feafe1ec514307ca"},
{file = "praw-7.0.0-py3-none-any.whl", hash = "sha256:74e4b6c3f206342d05272ce1770ac7b9c48207c9a7ffea3d5251460b70f18188"},
{file = "praw-7.0.0.tar.gz", hash = "sha256:65129169d560800261908415ed955f3cbc63648549820b3ccce0a823ffa2fd78"},
]
prawcore = [
{file = "prawcore-1.3.0-py3-none-any.whl", hash = "sha256:b907843ab969d759cbc03f1f749acea24d11859d6aed447b2fa1cd0eda9ecf34"},
{file = "prawcore-1.3.0.tar.gz", hash = "sha256:a982a49bc911fe0e3a9751319091c380f79d5e1ba1ba19cb8dbbce21ad8b0ca7"},
]
requests = [
{file = "requests-2.23.0-py2.py3-none-any.whl", hash = "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee"},
{file = "requests-2.23.0.tar.gz", hash = "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6"},
]
six = [
{file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
{file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
]
update-checker = [
{file = "update_checker-0.17-py2.py3-none-any.whl", hash = "sha256:1ff5dc7aab340b4f7710bd6c69d08ff5a5351617cd4ba0eb8886ddb285e2104f"},
{file = "update_checker-0.17.tar.gz", hash = "sha256:2def8db7f63bd45c7d19df5df570f3f3dfeb1a1f050869d7036529295db10e62"},
]
urllib3 = [
{file = "urllib3-1.25.9-py2.py3-none-any.whl", hash = "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115"},
{file = "urllib3-1.25.9.tar.gz", hash = "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527"},
]
websocket-client = [
{file = "websocket_client-0.57.0-py2.py3-none-any.whl", hash = "sha256:0fc45c961324d79c781bab301359d5a1b00b13ad1b10415a4780229ef71a5549"},
{file = "websocket_client-0.57.0.tar.gz", hash = "sha256:d735b91d6d1692a6a181f2a8c9e0238e5f6373356f561bb9dc4c7af36f452010"},
]

@ -19,7 +19,7 @@ def munge_idiot_data(reddit_dict):
every url that comes through my apparatus.
"""
protocol = 'https'
# pdb.set_trace()
for single_dict in reddit_dict:
if protocol in single_dict['url']:
pass
@ -108,13 +108,13 @@ def main(*args, **kwargs):
"--retag", "-rt", action='store_true',
help="Select this if you want to update an existing tag")
parser.add_argument(
"--reddit-un", "-run", required=True, help="Reddit username")
"--reddit-un", "-run", help="Reddit username")
parser.add_argument(
"--reddit-pw", "-rpw", required=True, help="Reddit password")
"--reddit-pw", "-rpw", help="Reddit password")
parser.add_argument(
"--reddit-cid", "-rcid", required=True, help="Reddit client id")
"--reddit-cid", "-rcid", help="Reddit client id")
parser.add_argument(
"--reddit-sec", "-rsec", required=True, help="Reddit client secret")
"--reddit-sec", "-rsec", help="Reddit client secret")
parser.add_argument(
"--pb-apikey", "-pba", required=True, help="Pinboard api key")
parsed = parser.parse_args()
@ -122,11 +122,11 @@ def main(*args, **kwargs):
sys.excepthook = idb_excepthook
LOGGER.setLevel(logging.DEBUG)
pinboard_token = parsed.pb_apikey
pinboard_base_url = "https://api.pinboard.in/v1/"
pinboard_auth_snippet = f"?auth_token={pinboard_token}"
if parsed.retag:
original_tag = input('what tag would you like to replace? ')
updated_tag = input('what tag would you like to use instead? ')
@ -135,6 +135,21 @@ def main(*args, **kwargs):
return response
if not parsed.reddit_un:
exit("hey man you need your reddit username (and maybe more)")
if not parsed.reddit_pw:
exit("hey man you need your reddit_pw (and maybe more)")
if not parsed.reddit_cid:
exit("hey man you need your reddit client id")
if not parsed.reddit_sec:
exit("hey man you need your reddit client secret")
if not parsed.pb_apikey:
exit("hey man you need to enter your pinboard API key")
reddit = praw.Reddit(client_id=parsed.reddit_cid,
client_secret=parsed.reddit_sec,
user_agent='/u/ pynit-tasks',
@ -147,17 +162,17 @@ def main(*args, **kwargs):
your_user = reddit.redditor(parsed.reddit_un)
saved_posts = your_user.saved(limit=None)
POSTS_TO_SAVE = []
posts_to_save = []
for link in saved_posts:
if hasattr(link, 'is_self'):
POSTS_TO_SAVE.append({
posts_to_save.append({
'title': link.title,
'tag': link.subreddit.display_name + ' added-by-pynnit',
'description': link.selftext,
'url': link.permalink
})
elif hasattr(link, 'is_root'):
POSTS_TO_SAVE.append({
posts_to_save.append({
'title': link.link_title,
'tag': link.subreddit.display_name + ' added-by-pynnit',
'description': link.body,
@ -166,9 +181,9 @@ def main(*args, **kwargs):
else:
print("shit is fucked.")
MUNGED_DATA = munge_idiot_data(POSTS_TO_SAVE)
munged_data = munge_idiot_data(posts_to_save)
with open('data.json', 'w') as outfile:
json.dump(MUNGED_DATA, outfile, indent=2)
json.dump(munged_data, outfile, indent=2)
# handle the pinboard side of things

@ -0,0 +1,17 @@
[tool.poetry]
name = "pynit"
version = "0.1.0"
description = "an integration between reddit.com and pinboard.in"
authors = ["josiah <me@jowj.net>"]
license = "unlicense"
[tool.poetry.dependencies]
python = "^3.7"
requests = "^2.23.0"
praw = "^7.0.0"
[tool.poetry.dev-dependencies]
[build-system]
requires = ["poetry>=0.12"]
build-backend = "poetry.masonry.api"
Loading…
Cancel
Save