From 35ed26468f19ec3e008c6194815e48eb4878ade5 Mon Sep 17 00:00:00 2001 From: Faerbit Date: Sat, 10 Apr 2021 14:04:16 +0200 Subject: [PATCH] Initial commit. --- .gitignore | 5 ++ Pipfile | 10 +++ Pipfile.lock | 103 ++++++++++++++++++++++++ config.py.example | 1 + docker-compose.yml | 191 +++++++++++++++++++++++++++++++++++++++++++++ main.py | 136 ++++++++++++++++++++++++++++++++ 6 files changed, 446 insertions(+) create mode 100644 .gitignore create mode 100644 Pipfile create mode 100644 Pipfile.lock create mode 100644 config.py.example create mode 100644 docker-compose.yml create mode 100755 main.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0af8fa6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +config.py +docker-compose.yaml +.idea +*.pyc +__pycache__ diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..ffa64f1 --- /dev/null +++ b/Pipfile @@ -0,0 +1,10 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +"ruamel.yaml" = "*" +requests = "*" + + diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..16ed896 --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,103 @@ +{ + "_meta": { + "hash": { + "sha256": "6c1d53da6279c12378d1c34b79989070350791a8b3a93ea85e49c0020a6d17ab" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c", + "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830" + ], + "version": "==2020.12.5" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "index": "pypi", + "version": "==2.25.1" + }, + "ruamel.yaml": { + "hashes": [ + "sha256:44bc6b54fddd45e4bc0619059196679f9e8b79c027f4131bb072e6a22f4d5e28", + "sha256:ac79fb25f5476e8e9ed1c53b8a2286d2c3f5dde49eb37dbcee5c7eb6a8415a22" + ], + "index": "pypi", + "version": "==0.17.4" + }, + "ruamel.yaml.clib": { + "hashes": [ + "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b", + "sha256:1236df55e0f73cd138c0eca074ee086136c3f16a97c2ac719032c050f7e0622f", + "sha256:1f8c0a4577c0e6c99d208de5c4d3fd8aceed9574bb154d7a2b21c16bb924154c", + "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91", + "sha256:28116f204103cb3a108dfd37668f20abe6e3cafd0d3fd40dba126c732457b3cc", + "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7", + "sha256:2fd336a5c6415c82e2deb40d08c222087febe0aebe520f4d21910629018ab0f3", + "sha256:30dca9bbcbb1cc858717438218d11eafb78666759e5094dd767468c0d577a7e7", + "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6", + "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6", + "sha256:46d6d20815064e8bb023ea8628cfb7402c0f0e83de2c2227a88097e239a7dffd", + "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0", + "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62", + "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99", + "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5", + "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026", + "sha256:6c0a5dc52fc74eb87c67374a4e554d4761fd42a4d01390b7e868b30d21f4b8bb", + "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2", + "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1", + "sha256:75f0ee6839532e52a3a53f80ce64925ed4aed697dd3fa890c4c918f3304bd4f4", + "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b", + "sha256:8be05be57dc5c7b4a0b24edcaa2f7275866d9c907725226cdde46da09367d923", + "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e", + "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c", + "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988", + "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f", + "sha256:b4b0d31f2052b3f9f9b5327024dc629a253a83d8649d4734ca7f35b60ec3e9e5", + "sha256:c6ac7e45367b1317e56f1461719c853fd6825226f45b835df7436bb04031fd8a", + "sha256:daf21aa33ee9b351f66deed30a3d450ab55c14242cfdfcd377798e2c0d25c9f1", + "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2", + "sha256:f6061a31880c1ed6b6ce341215336e2f3d0c1deccd84957b6fa8ca474b41e89f" + ], + "markers": "python_version < '3.10' and platform_python_implementation == 'CPython'", + "version": "==0.2.2" + }, + "urllib3": { + "hashes": [ + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.4" + } + }, + "develop": {} +} diff --git a/config.py.example b/config.py.example new file mode 100644 index 0000000..f649971 --- /dev/null +++ b/config.py.example @@ -0,0 +1 @@ +COMPOSE_FILE_PATH = diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..421ce5f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,191 @@ +version: '3' +volumes: + certs: + html: + vhostd: +networks: + nextcloud: + mariadb: +services: + nginx-proxy: + container_name: nginx-proxy + image: jwilder/nginx-proxy:alpine + restart: unless-stopped + ports: + - 80:80 + - 443:443 + environment: + - HSTS=max-age=31536000; includeSubDomains; preload + logging: + driver: journald + volumes: + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhostd:/etc/nginx/vhost.d + - /var/run/docker.sock:/tmp/docker.sock:ro + #- /docker/nextcloud/nginx-proxy.conf.hack:/etc/nginx/vhost.d/nc.faerb.it:ro + - /docker/nextcloud/nginx-proxy.conf:/etc/nginx/vhost.d/nc.faerb.it:ro + + letsencrypt: + container_name: letsencrypt + image: jrcs/letsencrypt-nginx-proxy-companion + restart: unless-stopped + environment: + - NGINX_PROXY_CONTAINER=nginx-proxy + logging: + driver: journald + volumes: + - certs:/etc/nginx/certs + - html:/usr/share/nginx/html + - vhostd:/etc/nginx/vhost.d + - /var/run/docker.sock:/var/run/docker.sock:ro + + nginx-page: + container_name: nginx-page + image: nginx:alpine + restart: unless-stopped + environment: + - VIRTUAL_PROTO=http + - VIRTUAL_HOST=faerb.it + - LETSENCRYPT_HOST=faerb.it + - LETSENCRYPT_EMAIL=faerbit@posteo.net + logging: + driver: journald + volumes: + - /docker/page:/usr/share/nginx/html:ro + networks: + - default + + nginx-nextcloud: + container_name: nginx-nextcloud + image: nginx:alpine + restart: unless-stopped + environment: + - VIRTUAL_PROTO=http + - VIRTUAL_HOST=nc.faerb.it + - LETSENCRYPT_HOST=nc.faerb.it + - LETSENCRYPT_EMAIL=faerbit@posteo.net + logging: + driver: journald + volumes: + - /docker/nextcloud/nginx.conf:/etc/nginx/nginx.conf:ro + - /docker/nextcloud/html:/var/www/html:ro + depends_on: + - nextcloud + networks: + - default + - nextcloud + + nextcloud: + container_name: nextcloud + image: library/nextcloud:21.0.1-fpm-alpine + restart: unless-stopped + env_file: + - /docker/mariadb/mariadb.env + environment: + - REDIS_HOST=redis + - MYSQL_HOST=mariadb + logging: + driver: journald + volumes: + - /docker/nextcloud/html:/var/www/html + depends_on: + - mariadb + - redis + networks: + - nextcloud + + nc-cron: + container_name: nc-cron + image: library/nextcloud:21.0.1-fpm-alpine + restart: unless-stopped + logging: + driver: journald + volumes: + - /docker/nextcloud/html:/var/www/html + - /docker/nextcloud/cron.sh:/cron.sh + entrypoint: /cron.sh + depends_on: + - mariadb + - redis + networks: + - nextcloud + + mariadb: + container_name: mariadb + image: mariadb:10.5 + restart: unless-stopped + ports: + - 127.0.0.1:3306:3306 + env_file: + - /docker/mariadb/mariadb.env + logging: + driver: journald + volumes: + - /docker/mariadb/mysql:/var/lib/mysql + networks: + - nextcloud + - mariadb + + redis: + container_name: redis + image: library/redis:6-alpine + restart: unless-stopped + logging: + driver: journald + networks: + - nextcloud + + gitea: + container_name: gitea + image: gitea/gitea:1.13 + restart: unless-stopped + ports: + - 22:22 + env_file: + - /docker/mariadb/gitea.env + - /docker/gitea/gitea.env + environment: + - VIRTUAL_HOST=git.faerb.it + - VIRTUAL_PORT=3000 + - LETSENCRYPT_HOST=git.faerb.it + - LETSENCRYPT_EMAIL=faerbit@posteo.net + - SSH_DOMAIN=git.faerb.it + - ROOT_URL=git.faerb.it + - DB_TYPE=mysql + - DB_HOST=mariadb:3306 + - RUN_MODE=prod + - DISABLE_REGISTRATION=true + logging: + driver: journald + volumes: + - /docker/gitea/data:/data + networks: + - default + - mariadb + depends_on: + - mariadb + + immoscrape: + container_name: immoscrape + image: registry.gitlab.com/faerbit/immoscrape:main + restart: unless-stopped + env_file: + - /docker/mariadb/immoscrape.env + environment: + - VIRTUAL_HOST=ims.faerb.it + - VIRTUAL_PROTO=uwsgi + - VIRTUAL_PORT=3000 + - LETSENCRYPT_HOST=ims.faerb.it + - LETSENCRYPT_EMAIL=faerbit@posteo.net + - IMMOSCRAPE_TELEGRAM_SEND_CONFIG_PATH=/app/config/telegram-send.conf + logging: + driver: journald + volumes: + - /docker/immoscrape/telegram-send.conf:/app/config/telegram-send.conf + - /etc/localtime:/etc/localtime:ro + networks: + - default + - mariadb + depends_on: + - mariadb diff --git a/main.py b/main.py new file mode 100755 index 0000000..b8b621f --- /dev/null +++ b/main.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +import re +import sys + +import requests + +from distutils.version import LooseVersion + +from ruamel.yaml import YAML + +from config import COMPOSE_FILE_PATH +REGISTRY_URL = "https://registry.hub.docker.com/v2/" + +RED = "\033[31m" +BLU = "\033[34m" +WHT = "\033[37m" +GRY = "\033[90m" +NC = "\033[0m" + + +class DockerAuth(requests.auth.AuthBase): + def __init__(self): + self.auth_token = None + + def __call__(self, request): + if self.auth_token: + request.headers["Authorization"] = f"Bearer {self.auth_token}" + request.register_hook('response', self.handle_401) + return request + + def handle_401(self, response, **kwargs): + if not 400 <= response.status_code < 500: + return response + if "Www-Authenticate" not in response.headers: + return response + auth_header = response.headers["Www-Authenticate"] + regex = re.compile(r'(\w+)[=]\"([\w:/.]+)\"') + auth_data = dict(regex.findall(auth_header)) + if "realm" not in auth_data or \ + "service" not in auth_data or \ + "scope" not in auth_data: + exit_msg("Unexpected auth header") + params = { + "service": auth_data["service"], + "client_id": "fuck-your-stupid-undocumented-shit", + "scope": auth_data["scope"] + } + print(f"{GRY}Request to {auth_data['realm']}{NC}") + resp = requests.get(auth_data["realm"], params=params) + self.auth_token = resp.json()["token"] + print(f"{GRY}Request to {response.url}{NC}") + response = requests.get(response.url, headers={"Authorization": f"Bearer {self.auth_token}"}) + return response + + +def exit_msg(msg, exit_code=1): + if exit_code > 0: + COLOR = RED + else: + COLOR = BLU + print(f"{COLOR}{msg}{NC}", file=sys.stderr) + sys.exit(exit_code) + + +def main(): + loader = YAML() + with open(COMPOSE_FILE_PATH) as f: + yaml_data = loader.load(f) + if "services" not in yaml_data: + exit_msg("Invalid compose file. No services") + summary = {} + service_length = 0 + old_length = 0 + new_length = 0 + for k, v in yaml_data["services"].items(): + if "image" not in v: + exit_msg("Invalid compose file. No image") + image_str = v["image"] + if not any(char.isdigit() for char in image_str): + continue + image_name, image_version = image_str.split(":") + print(f'Working on service "{k}" with image "{image_name}": {image_version}') + if "-" in image_version: + i = image_version.find("-") + pure_version = image_version[:i] + variant = image_version[i:] + else: + pure_version = image_version + variant = "" + specificity = pure_version.count(".") + old_version = LooseVersion(pure_version) + regex_str = ".".join([r"\d+"] * (specificity + 1)) + variant + "$" + regex = re.compile(regex_str) + if "/" not in image_name: + image_name = "library/" + image_name + url = REGISTRY_URL + image_name + "/tags/list" + print(f"{GRY}Request to {url}{NC}") + resp = requests.get(url, auth=DockerAuth()) + tags = resp.json()["tags"] + newest_full_version = image_version + newest_version = old_version + for tag in tags: + if not regex.match(tag): + continue + if variant: + new_pure_version = tag[:-len(variant)] + else: + new_pure_version = tag + new_version = LooseVersion(new_pure_version) + if new_version > old_version: + newest_version = new_version + newest_full_version = tag + if newest_version != old_version: + print(f"Found newer version: {newest_full_version}") + summary[k] = (image_version, newest_full_version) + service_length = max(service_length, len(k)) + old_length = max(old_length, len(image_version)) + new_length = max(new_length, len(newest_full_version)) + yaml_data["services"][k]["image"] = f"{image_name}:{newest_full_version}" + if not summary: + exit_msg("No new versions.", 0) + print(f"{WHT}Summary:{NC}") + for service, (old, new) in summary.items(): + print(f'{WHT}{service:>{service_length}}: {old:>{old_length}} -> {new:>{new_length}}{NC}') + user_input = input("Confirm ? [y/N] ") + if user_input.lower() != "y": + exit_msg("User did not confirm. Exit", 0) + loader.indent(mapping=2, sequence=4, offset=2) + with open(COMPOSE_FILE_PATH, "w") as f: + loader.dump(yaml_data, f) + exit_msg(f'Successfully written to file "{COMPOSE_FILE_PATH}"', 0) + + +if __name__ == "__main__": + main()