diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | clip2file.py | 123 | ||||
| -rwxr-xr-x | dlr34.py | 302 |
3 files changed, 426 insertions, 1 deletions
@@ -1,4 +1,4 @@ -INSTALL_LIST=autoDHCP.sh autoWall.sh cmount.sh coWifi.sh cumount.sh imgs2pdf.sh light2.sh toMp3.sh bot4chan.py track.py autoMMS.py pyhttpd.py +INSTALL_LIST=autoDHCP.py autoWall.sh cmount.sh coWifi.sh cumount.sh imgs2pdf.sh light2.sh toMp3.sh bot4chan.py track.py autoMMS.py pyhttpd.py clip2file.py install: @if [ -z $(filter-out $@,$(MAKECMDGOALS)) ] ; then \ diff --git a/clip2file.py b/clip2file.py new file mode 100644 index 0000000..d62b725 --- /dev/null +++ b/clip2file.py @@ -0,0 +1,123 @@ +#!/bin/env python + +import subprocess +import datetime +import sys +import os +import argparse + + +""" + This program will write the clipboard to a file. + The format of the file is determined by the available targets and the + priority list. +""" + +OUTPUT_DIR = "~" + +def get_clipboard_content(target): + # Get the clipboard contents. + try: + clipfile = subprocess.check_output( + ['xclip', '-o', '-t', target, '-selection', 'clipboard'] + ) + except subprocess.CalledProcessError as e: + print(f"Failed to get clipboard contents: {e}", file=sys.stderr) + sys.exit(e.returncode) + # Check if the clipboard contents are empty. + if not clipfile: + print("Clipboard is empty.", file=sys.stderr) + sys.exit(1) + + return clipfile + +def get_available_targets(): + # Get the lists of available targets. + # Executes `xclip -o -t TARGETS -selection clipboard` and get the output + # as a list of strings. + try: + output = subprocess.check_output( + ['xclip', '-o', '-t', 'TARGETS', '-selection', 'clipboard'] + ) + except subprocess.CalledProcessError as e: + print(f"xclip failed: {e}", file=sys.stderr) + sys.exit(e.returncode) + + return output.decode('utf-8').split('\n') + # The output is a list of strings, each string is a target. + +def save_file(clipfile, filename, output_dir, ext): + # Write the clipboard contents to a file. + pathfile = os.path.expanduser(f"{output_dir}/{filename}.{ext}") + + try: + with open(pathfile, 'wb') as f: + try: + f.write(clipfile) + except IOError as e: + print(f"Failed to write to file {filename} in {output_dir}: {e}", file=sys.stderr) + sys.exit(e.errno) + except IOError as e: + print(f"Failed to open file {filename} in {output_dir}: {e}", file=sys.stderr) + + +def main(filename, output_dir): + # The priority of the target. + # The first target in the list is the most important. + # The final target will be the first of this list matching the available targets + priority = [ + ("image/avif", "avif"), + ("image/png", "png"), + ("text/plain", "txt"), + ("UTF8_STRING", "txt"), + ] + + targets = get_available_targets() + + # Get the first target in the priority list that is available. + # If no target is found, the script exits. + target = None + ext = None + for f, e in priority: + if f in targets: + target, ext = f, e + break + + if target is None or not isinstance(target, str): + print("No supported clipboard target found.", file=sys.stderr) + sys.exit(1) + + clipfile = get_clipboard_content(target) + + save_file(clipfile, filename, output_dir, ext) + + print(f"Saved clipboard contents to {output_dir}/{filename}.{ext}") + + +if __name__ == '__main__': + date = datetime.datetime.now().isoformat() + default_filename = f"{date}_clip" + + parser = argparse.ArgumentParser( + description="Save clipboard contents to a file." + ) + # Select the directory + parser.add_argument( + '-d', '--directory', + help="Directory to save the clipboard contents to.", + default=OUTPUT_DIR, + type=str + ) + # Select the filename + parser.add_argument( + '-f', '--filename', + help="Filename to save the clipboard contents to.", + default=default_filename, + type=str + ) + + args = parser.parse_args() + + main(args.filename, args.directory) + + diff --git a/dlr34.py b/dlr34.py new file mode 100755 index 0000000..5045274 --- /dev/null +++ b/dlr34.py @@ -0,0 +1,302 @@ +#!/bin/env python + +import os +import re +import requests +import sys +import csv +import lxml +import json +from urllib.parse import urlparse +from pathlib import Path +import lxml.html +import argparse + + +DEBUG_MODE = True +dirOut = Path("r34") + + +class Spinner(object): + def __init__(self): + self.SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'] + self.SPINNER_status = 1 + + def __call__(self): + self.SPINNER_status = (self.SPINNER_status + 1) % len(self.SPINNER) + return self.SPINNER[self.SPINNER_status] + + +quickSpinner = Spinner() + + +def pretty_print_Request_header(req): + print(f"{req.method} {req.url}") + + for k, v in req.headers.items(): + print(f"{k}: {v}") + + +def outputFileFromURL(url): + u = urlparse(url) + name = Path(u.path).name + alreadyExists = False + + finalDest = dirOut.joinpath(Path(name)) + + i = 1 + + while finalDest.exists(): + alreadyExists = True + + s_name = Path(u.path).stem + s_suffix = Path(u.path).suffix + + next_name = f"{s_name}_{i}{s_suffix}" + i += 1 + + finalDest = dirOut.joinpath(Path(next_name)) + + return alreadyExists, str(finalDest) + + +def downloadURL(s, url, idLink, output, headers={}): + r = s.get(url, stream=True, headers=headers) + + u = urlparse(url) + + if r.ok: + if DEBUG_MODE: + print(f"\n{url} >> {output}") + + print(f"\r⬇️ {Path(u.path).name}", end='') + + try: + with open(output, "wb") as f: + print(f"\r⬇️ {quickSpinner()} {Path(u.path).name}", end='') + chunk_size = 4 * 1024 + + for chunk in r.iter_content(chunk_size=chunk_size): + f.write(chunk) + print(f"\rDownloaded {Path(u.path).name} - {idLink}") + except Exception as e: + print(f"\r☠️ error {Path(u.path).name} - {idLink}") + if DEBUG_MODE: + print(e) + +def downloadFromItems(items, s, u): + replace = { + 'mov480.': 'mov.', + 'mov720.': 'mov.', + 'mov256.': 'mov.', + 'pic256.': 'pic.', + 'pic480.': 'pic.', + 'pic720.': 'pic.', + 'picsmall.': 'pic.' + } + + for item in items: + isDownloaded = False + isVideo = item['duration'] is not None + for file in item['imageLinks']: + url = file['url'] + if isVideo and not url.endswith('.mp4'): + continue + + for r,t in replace.items(): + if r in url: + url = url.replace(r, t) + e, outputFile = outputFileFromURL(url) + if e: + print("Already downloaded") + return + + headers = {"Referer": f"{u.scheme}://{u.hostname}/"} + headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" + headers['Accept-encoding'] = "gzip, deflate, br" + downloadURL(s, url, "???", outputFile, headers) + isDownloaded = True + break + if isDownloaded: + break + + +def downloadImagesFromPlaylist(index, link): + s = requests.Session() + + u = urlparse(link) + p = Path(u.path) + playlistId = p.stem + + print(f"Link n°{index} - {playlistId}") + + nbPage = 0 + while True: + urlPlaylistFormat = f"https://rule34.world/api/playlist-item?playlistId={playlistId}&Skip={nbPage*60}&Take=60&DisableTotal=true" + resp = s.get(urlPlaylistFormat) + if resp.ok and resp.status_code == 200: + data = json.loads(resp.content) + if len(data['items']) > 0: + print(f"Ok ! {len(data['items'])}") + downloadFromItems(data['items'], s, u) + else: + break + + nbPage += 1 + + return + try: + html = lxml.html.fromstring(resp.content) + boxes = html.cssselect("div.box a.boxInner") + for box in boxes: + url = box.attrib['href'] + url = f"{u.scheme}://{u.hostname}{url}" + downloadImageFromPage(index, url) + except Exception as e: + print(f"\r☠️ error {Path(u.path).name} - {idLink}") + if DEBUG_MODE: + print(e) + + +def downloadImagesFromUserboard(index, link): + s = requests.Session() + + u = urlparse(link) + p = Path(u.path) + boardName = p.stem + + print(f"Link n°{index} - {boardName}", end='') + resp = s.get(link) + + findImagesRegex = re.compile('<img class="image" src="/images/t.png" data-src="(.+)\?width=300"') + if resp.ok and resp.status_code == 200: + for subIndex, urlImg in enumerate(findImagesRegex.findall(resp.content.decode('utf8'))): + match = re.search(r'(\d{7,})', urlImg) + if not match: + return + + e, outputFile = outputFileFromURL(urlImg) + if e: + print("Already downloaded") + return + + if DEBUG_MODE: + print(f"\n{urlImg} >> {outputFile}") + + idLink = match.group(1) + print(f"Link n°{index}.{subIndex} - {boardName}>{idLink}", end='') + + headers = {"Referer": f"{u.scheme}://{u.hostname}/"} + headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" + headers['Accept-encoding'] = "gzip, deflate, br" + downloadURL(s, urlImg, idLink, outputFile, headers) + + +def downloadVideoFromSource(index, urlBase, idVid, urlVideo): + s = requests.Session() + + print(f"Link n°{index} - Vidéo {idVid}", end='') + print(f"Link : {urlVideo} !", end='\n') + + + u = urlparse(urlBase) + urlVideo = urlVideo.replace('mov480', 'mov') + + e, outputFile = outputFileFromURL(urlVideo) + if e: + print("Already downloaded") + return + + headers = {"Referer": f"{u.scheme}://{u.hostname}/"} + headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" + headers['Accept-encoding'] = "gzip, deflate, br" + + print(f"Download to {outputFile}") + downloadURL(s, urlVideo, idVid, outputFile, headers) + + +def downloadImageFromPage(index, link): + s = requests.Session() + match = re.search(r'(\d{6,})', link) + + if not match: + return + + idLink = match.group(1) + print(f"Link n°{index} - {idLink}\n", end='') + resp = s.get(link) + + if resp.ok and resp.status_code == 200: + try: + html = lxml.html.fromstring(resp.content) + imgs = html.cssselect("img.img.shadow-base") + + if len(imgs) == 0: + vids = html.cssselect("video.video.shadow-base source") + if len(vids) > 0: + url = vids[0].attrib['src'] + + u = urlparse(link) + if url[0] == '/': + url = f"{u.scheme}://{u.hostname}{url}" + + downloadVideoFromSource(index, link, idLink, url) + else: + img = imgs[0] + + if 'src' in img.attrib: + url = img.attrib['src'] + u = urlparse(link) + if url[0] == '/': + url = f"{u.scheme}://{u.hostname}{url}" + + url = url.replace('picsmall', 'pic') + + e, outputFile = outputFileFromURL(url) + if e: + print("Already downloaded") + return + + headers = {"Referer": f"{u.scheme}://{u.hostname}/"} + headers['Accept'] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8" + headers['Accept-encoding'] = "gzip, deflate, br" + downloadURL(s, url, idLink, outputFile, headers) + + except: + print(f"\r☠️ error {Path(u.path).name} - {idLink}") + if DEBUG_MODE: + print(e) + return + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-d", "--directory", help="Output directory") + parser.add_argument('files', nargs=argparse.REMAINDER) + args = parser.parse_args() + + if args.directory: + global dirOut + dirOut = Path(args.directory) + + if not dirOut.exists(): + print(f"Output dir \"{dirOut}\" doesn't exists") + return + + if not dirOut.is_dir(): + print(f"Output dir \"{dirOut}\" isn't a directory") + return + + for file in args.files: + with open(file) as f_listLink: + for index, link in enumerate(f_listLink.read().split()): + if '/post/' in link: + downloadImageFromPage(index, link) + if '/user/' in link: + downloadImagesFromUserboard(index, link) + if '/playlists/' in link: + downloadImagesFromPlaylist(index, link) + + +if __name__ == "__main__": + main() |