irongit
Sign inRegister
huncholanehuncholane/django-smartwatchPublic

Reusable Django app for system monitoring. Built to make dev environment more like production environment. Combines celery with gunicorn and daphne.

initial commit

huncholanehuncholaneauthored
commit 7ddf2cc17c0ad7ba1bc013c89fe6017cd72d8184Browse files

47 files changed, +513 -0

+8-0django_smartwatch.egg-info/PKG-INFO
@@ -0,0 +1,8 @@
1+Metadata-Version: 2.4
2+Name: django-smartwatch
3+Version: 0.1.0
4+Summary: Reusable Django app for system monitoring
5+Project-URL: Homepage, https://github.com/huncholane/django-smartwatch
6+Requires-Python: >=3.8
7+Description-Content-Type: text/markdown
8+Requires-Dist: django
+24-0django_smartwatch.egg-info/SOURCES.txt
@@ -0,0 +1,24 @@
1+pyproject.toml
2+django_smartwatch.egg-info/PKG-INFO
3+django_smartwatch.egg-info/SOURCES.txt
4+django_smartwatch.egg-info/dependency_links.txt
5+django_smartwatch.egg-info/requires.txt
6+django_smartwatch.egg-info/top_level.txt
7+smartwatch/__init__.py
8+smartwatch/admin.py
9+smartwatch/apps.py
10+smartwatch/models.py
11+smartwatch/settings.py
12+smartwatch/tasks.py
13+smartwatch/tests.py
14+smartwatch/views.py
15+smartwatch/watch.py
16+smartwatch/management/commands/printenv.py
17+smartwatch/management/commands/smarttest.py
18+smartwatch/management/commands/smartwatch.py
19+smartwatch/management/commands/startcelery.py
20+smartwatch/migrations/0001_initial.py
21+smartwatch/migrations/0002_alter_filehash_file_path.py
22+smartwatch/migrations/0003_rename_filehash_smartfile.py
23+smartwatch/migrations/0004_delete_smartfile.py
24+smartwatch/migrations/__init__.py
+1-0django_smartwatch.egg-info/dependency_links.txt
@@ -0,0 +1 @@
1+
+1-0django_smartwatch.egg-info/requires.txt
@@ -0,0 +1 @@
1+django
+1-0django_smartwatch.egg-info/top_level.txt
@@ -0,0 +1 @@
1+smartwatch
+14-0pyproject.toml
@@ -0,0 +1,14 @@
1+[project]
2+name = "django-smartwatch"
3+version = "0.1.0"
4+description = "Reusable Django app for system monitoring"
5+readme = "README.md"
6+requires-python = ">=3.8"
7+dependencies = ["django"]
8+
9+[project.urls]
10+Homepage = "https://github.com/huncholane/django-smartwatch"
11+
12+[build-system]
13+requires = ["setuptools"]
14+build-backend = "setuptools.build_meta"
+0-0smartwatch/__init__.py

No content changes (mode or rename only).

+0-0smartwatch/__pycache__/__init__.cpython-312.pyc

Binary file changed.

+0-0smartwatch/__pycache__/__init__.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/admin.cpython-312.pyc

Binary file changed.

+0-0smartwatch/__pycache__/admin.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/apps.cpython-312.pyc

Binary file changed.

+0-0smartwatch/__pycache__/apps.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/celery.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/middleware.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/models.cpython-312.pyc

Binary file changed.

+0-0smartwatch/__pycache__/models.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/settings.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/tasks.pypy310.pyc

Binary file changed.

+0-0smartwatch/__pycache__/tests.pypy310.pyc

Binary file changed.

+6-0smartwatch/apps.py
@@ -0,0 +1,6 @@
1+from django.apps import AppConfig
2+
3+
4+class SmartwatchConfig(AppConfig):
5+ default_auto_field = 'django.db.models.BigAutoField'
6+ name = 'smartwatch'
+0-0smartwatch/management/commands/__pycache__/smarttest.pypy310.pyc

Binary file changed.

+0-0smartwatch/management/commands/__pycache__/smartwatch.pypy310.pyc

Binary file changed.

+0-0smartwatch/management/commands/__pycache__/startcelery.pypy310.pyc

Binary file changed.

+11-0smartwatch/management/commands/printenv.py
@@ -0,0 +1,11 @@
1+import os
2+from django.core.management.base import BaseCommand
3+
4+
5+class Command(BaseCommand):
6+ help = "Print environment variables in alphabetical order"
7+
8+ def handle(self, *args, **kwargs):
9+ env_vars = sorted(os.environ.items())
10+ for key, value in env_vars:
11+ self.stdout.write(f"\033[1;36m{key}\033[0m: {value}")
+78-0smartwatch/management/commands/smarttest.py
@@ -0,0 +1,78 @@
1+import os
2+import time
3+from multiprocessing import Process
4+from typing import Any, Iterable
5+
6+from django.core.management.commands.test import Command as TestCommand
7+from watchdog.events import FileSystemEventHandler
8+from watchdog.observers import Observer
9+
10+
11+class Command(TestCommand):
12+ help = "Watch test files and run tests when they change. Runs the testing in another process."
13+ args: Iterable[Any]
14+ options: dict
15+
16+ def handle(self, *args, **options):
17+ self.args = args
18+ self.options = options
19+
20+ event_handler = TestFileChangeHandler(self)
21+ observer = Observer()
22+ observer.schedule(event_handler, os.getcwd(), recursive=True)
23+ observer.start()
24+
25+ try:
26+ while True:
27+ time.sleep(1)
28+ except KeyboardInterrupt:
29+ observer.stop()
30+ observer.join()
31+
32+
33+class TestFileChangeHandler(FileSystemEventHandler):
34+ process: Process
35+
36+ def __init__(self, command: Command):
37+ self.command = command
38+ self.command.stdout.write(
39+ self.command.style.SUCCESS("Running the tests for the first time...")
40+ )
41+ # self.process = None
42+ self.run_tests()
43+
44+ def on_modified(self, event):
45+ assert isinstance(event.src_path, str)
46+ if event.src_path.endswith(".py"):
47+ self.command.stdout.write(
48+ self.command.style.SUCCESS("File change detected. Retesting...")
49+ )
50+ if self.process:
51+ self.process.kill()
52+ self.run_tests()
53+ self.command.stdout.write(
54+ self.command.style.SUCCESS("Watching for changes to retest...")
55+ )
56+
57+ def run_tests(self):
58+ self.process = Process(
59+ target=start_tests, args=self.command.args, kwargs=self.command.options
60+ )
61+ self.process.start()
62+ self.process.join()
63+
64+
65+def start_tests(*args, **_):
66+ import dotenv
67+
68+ dotenv.load_dotenv("smarttest.env")
69+ try:
70+ from django.core.management import execute_from_command_line
71+ except ImportError as exc:
72+ raise ImportError(
73+ "Couldn't import Django. Are you sure it's installed and "
74+ "available on your PYTHONPATH environment variable? Did you "
75+ "forget to activate a virtual environment?"
76+ ) from exc
77+ args = ["manage.py", "test", "--keepdb", *args]
78+ execute_from_command_line(args)
+10-0smartwatch/management/commands/smartwatch.py
@@ -0,0 +1,10 @@
1+from django.core.management.base import BaseCommand
2+
3+from smartwatch.watch import watch_server
4+
5+
6+class Command(BaseCommand):
7+ help = "Starts the file watcher"
8+
9+ def handle(self, *args, **options):
10+ watch_server()
+20-0smartwatch/management/commands/startcelery.py
@@ -0,0 +1,20 @@
1+import signal
2+
3+from django.core.management.base import BaseCommand
4+from psutil import Popen
5+
6+from smartwatch import settings
7+
8+
9+class Command(BaseCommand):
10+ help = "Starts the celery worker"
11+ celery_process: Popen | None = None
12+
13+ def handle(self, *args, **kwargs):
14+ signal.signal(signal.SIGINT, self.shutdown)
15+ self.celery_process = Popen(["celery", "-A", settings.PROJECT_NAME, "worker"])
16+ self.celery_process.wait()
17+
18+ def shutdown(self, *_):
19+ if self.celery_process:
20+ self.celery_process.send_signal(signal.SIGINT)
+29-0smartwatch/migrations/0001_initial.py
@@ -0,0 +1,29 @@
1+# Generated by Django 5.0.7 on 2024-10-31 11:38
2+
3+from django.db import migrations, models
4+
5+
6+class Migration(migrations.Migration):
7+
8+ initial = True
9+
10+ dependencies = []
11+
12+ operations = [
13+ migrations.CreateModel(
14+ name="FileHash",
15+ fields=[
16+ (
17+ "id",
18+ models.BigAutoField(
19+ auto_created=True,
20+ primary_key=True,
21+ serialize=False,
22+ verbose_name="ID",
23+ ),
24+ ),
25+ ("file_path", models.CharField(max_length=1000)),
26+ ("md5_hash", models.CharField(max_length=32)),
27+ ],
28+ ),
29+ ]
+38-0smartwatch/migrations/0002_alter_filehash_file_path.py
@@ -0,0 +1,38 @@
1+# Generated by Django 5.0.7 on 2024-10-31 12:44
2+
3+import logging
4+from django.db import migrations, models
5+
6+
7+def handle_duplicates(apps, _):
8+ try:
9+ FileHash = apps.get_model("smartwatch", "FileHash")
10+ duplicates = (
11+ FileHash.objects.values("file_path")
12+ .annotate(count=models.Count("id"))
13+ .filter(count__gt=1)
14+ )
15+
16+ for duplicate in duplicates:
17+ file_path = duplicate["file_path"]
18+ duplicate_entries = FileHash.objects.filter(file_path=file_path)
19+ # Keep the first entry and delete the rest
20+ first_entry = duplicate_entries.first()
21+ duplicate_entries.exclude(id=first_entry.id).delete()
22+ except Exception:
23+ logging.error("Error while handling duplicates")
24+
25+
26+class Migration(migrations.Migration):
27+ dependencies = [
28+ ("smartwatch", "0001_initial"),
29+ ]
30+
31+ operations = [
32+ migrations.RunPython(handle_duplicates),
33+ migrations.AlterField(
34+ model_name="filehash",
35+ name="file_path",
36+ field=models.CharField(max_length=1000, unique=True),
37+ ),
38+ ]
+17-0smartwatch/migrations/0003_rename_filehash_smartfile.py
@@ -0,0 +1,17 @@
1+# Generated by Django 5.0.7 on 2024-11-02 22:47
2+
3+from django.db import migrations
4+
5+
6+class Migration(migrations.Migration):
7+
8+ dependencies = [
9+ ("smartwatch", "0002_alter_filehash_file_path"),
10+ ]
11+
12+ operations = [
13+ migrations.RenameModel(
14+ old_name="FileHash",
15+ new_name="SmartFile",
16+ ),
17+ ]
+16-0smartwatch/migrations/0004_delete_smartfile.py
@@ -0,0 +1,16 @@
1+# Generated by Django 5.0.7 on 2024-12-04 21:55
2+
3+from django.db import migrations
4+
5+
6+class Migration(migrations.Migration):
7+
8+ dependencies = [
9+ ("smartwatch", "0003_rename_filehash_smartfile"),
10+ ]
11+
12+ operations = [
13+ migrations.DeleteModel(
14+ name="SmartFile",
15+ ),
16+ ]
+0-0smartwatch/migrations/__init__.py

No content changes (mode or rename only).

+0-0smartwatch/migrations/__pycache__/0001_initial.cpython-312.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0001_initial.pypy310.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0002_alter_filehash_file_path.cpython-312.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0002_alter_filehash_file_path.pypy310.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0003_rename_filehash_smartfile.cpython-312.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0003_rename_filehash_smartfile.pypy310.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0004_delete_smartfile.cpython-312.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/0004_delete_smartfile.pypy310.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/__init__.cpython-312.pyc

Binary file changed.

+0-0smartwatch/migrations/__pycache__/__init__.pypy310.pyc

Binary file changed.

+1-0smartwatch/models.py
@@ -0,0 +1 @@
1+
+64-0smartwatch/settings.py
@@ -0,0 +1,64 @@
1+from django.conf import settings
2+from environs import Env
3+
4+env = Env()
5+
6+# Gunicorn settings
7+SMARTWATCH_GUNICORN_PORT = getattr(
8+ settings, "SMARTWATCH_GUNICORN_PORT", env.int("SMARTWATCH_GUNICORN_PORT", 8000)
9+)
10+SMARTWATCH_GUNICORN_HOST = getattr(
11+ settings, "SMARTWATCH_GUNICORN_HOST", env.str("SMARTWATCH_GUNICORN_HOST", "0.0.0.0")
12+)
13+SMARTWATCH_USE_GUNICORN = getattr(
14+ settings, "SMARTWATCH_USE_GUNICORN", env.bool("SMARTWATCH_USE_GUNICORN", True)
15+)
16+SMARTWATCH_GUNICORN_WORKERS = getattr(
17+ settings, "SMARTWATCH_GUNICORN_WORKERS", env.int("SMARTWATCH_GUNICORN_WORKERS", 4)
18+)
19+SMARTWATCH_GUNICORN_LOGLEVEL = getattr(
20+ settings,
21+ "SMARTWATCH_GUNICORN_LOGLEVEL",
22+ env.str("SMARTWATCH_GUNICORN_LOGLEVEL", "warning"),
23+)
24+
25+# Daphne settings
26+SMARTWATCH_DAPHNE_PORT = getattr(
27+ settings, "SMARTWATCH_DAPHNE_PORT", env.int("SMARTWATCH_DAPHNE_PORT", 8001)
28+)
29+SMARTWATCH_DAPHNE_HOST = getattr(
30+ settings, "SMARTWATCH_DAPHNE_HOST", env.str("SMARTWATCH_DAPHNE_HOST", "0.0.0.0")
31+)
32+SMARTWATCH_USE_DAPHNE = getattr(
33+ settings, "SMARTWATCH_USE_DAPHNE", env.bool("SMARTWATCH_USE_DAPHNE", True)
34+)
35+
36+# Celery settings
37+SMARTWATCH_START_CELERY = getattr(
38+ settings, "SMARTWATCH_START_CELERY", env.bool("SMARTWATCH_START_CELERY", True)
39+)
40+SMARTWATCH_CELERY_LOGLEVEL = getattr(
41+ settings,
42+ "SMARTWATCH_CELERY_LOGLEVEL",
43+ env.str("SMARTWATCH_CELERY_LOGLEVEL", "info"),
44+)
45+
46+# Other settings
47+SMARTWATCH_COLLECT_STATIC = getattr(
48+ settings, "SMARTWATCH_COLLECT_STATIC", env.bool("SMARTWATCH_COLLECT_STATIC", True)
49+)
50+SMARTWATCH_MIGRATE = getattr(
51+ settings, "SMARTWATCH_MIGRATE", env.bool("SMARTWATCH_MIGRATE", True)
52+)
53+SMARTWATCH_TEST_DB_HOST = getattr(
54+ settings, "SMARTWATCH_TEST_DB_HOST", env.str("SMARTWATCH_TEST_DB_HOST", "localhost")
55+)
56+SMARTWATCH_TEST_DB_PORT = getattr(
57+ settings, "SMARTWATCH_TEST_DB_PORT", env.int("SMARTWATCH_TEST_DB_PORT", 5432)
58+)
59+SMARTWATCH_DEBOUNCE_SECONDS = getattr(
60+ settings,
61+ "SMARTWATCH_DEBOUNCE_SECONDS",
62+ env.float("SMARTWATCH_DEBOUNCE_SECONDS", 0.1),
63+)
64+PROJECT_NAME = getattr(settings, "PROJECT_NAME", env.str("PROJECT_NAME"))
+19-0smartwatch/tasks.py
@@ -0,0 +1,19 @@
1+from celery import shared_task
2+import os
3+from smartwatch import settings
4+
5+
6+@shared_task
7+def start_asgi(hostname=None, port=None):
8+ hostname = hostname or settings.SMARTWATCH_DAPHNE_HOST
9+ port = port or settings.SMARTWATCH_DAPHNE_PORT
10+ os.system(
11+ f"daphne -b {hostname} -p {port} {settings.PROJECT_NAME}.asgi:application"
12+ )
13+
14+
15+@shared_task
16+def start_wsgi(hostname=None, port=None):
17+ hostname = hostname or settings.SMARTWATCH_GUNICORN_HOST
18+ port = port or settings.SMARTWATCH_GUNICORN_PORT
19+ os.system(f"gunicorn -b {hostname}:{port} {settings.PROJECT_NAME}.wsgi:application")
+155-0smartwatch/watch.py
@@ -0,0 +1,155 @@
1+import logging
2+import signal
3+import time
4+from subprocess import DEVNULL
5+from threading import Thread
6+
7+from psutil import Popen
8+from watchdog.events import RegexMatchingEventHandler
9+from watchdog.observers import Observer
10+
11+from smartwatch import settings
12+
13+
14+def create_collectstatic():
15+ return Popen(
16+ ["python", "manage.py", "collectstatic", "--noinput"],
17+ )
18+
19+
20+def create_migrate():
21+ return Popen(["python", "manage.py", "migrate"], stdout=DEVNULL, stderr=DEVNULL)
22+
23+
24+def create_gunicorn():
25+ return Popen(
26+ [
27+ "gunicorn",
28+ f"--bind={settings.SMARTWATCH_GUNICORN_HOST}:{settings.SMARTWATCH_GUNICORN_PORT}",
29+ f"--workers={settings.SMARTWATCH_GUNICORN_WORKERS}",
30+ f"{settings.PROJECT_NAME}.wsgi:application",
31+ f"--log-level={settings.SMARTWATCH_GUNICORN_LOGLEVEL}",
32+ ]
33+ )
34+
35+
36+def create_daphne():
37+ return Popen(
38+ [
39+ "daphne",
40+ "-b",
41+ "0.0.0.0",
42+ "-p",
43+ str(settings.SMARTWATCH_DAPHNE_PORT),
44+ f"{settings.PROJECT_NAME}.asgi:application",
45+ ]
46+ )
47+
48+
49+def create_celery():
50+ return Popen(
51+ [
52+ "celery",
53+ "-A",
54+ settings.PROJECT_NAME,
55+ "worker",
56+ f"--loglevel={settings.SMARTWATCH_CELERY_LOGLEVEL}",
57+ ]
58+ )
59+
60+
61+def create_install():
62+ return Popen(
63+ ["pip", "install", "-r", "requirements.txt"],
64+ stdout=DEVNULL,
65+ stderr=DEVNULL,
66+ )
67+
68+
69+def prep_server():
70+ create_install().wait()
71+ logging.info("Requirements installed")
72+ if settings.SMARTWATCH_MIGRATE:
73+ create_migrate().wait()
74+ logging.info("Migrated database")
75+ if settings.SMARTWATCH_COLLECT_STATIC:
76+ create_collectstatic().wait()
77+ logging.info("Collected static files")
78+
79+
80+class ServerHandler(RegexMatchingEventHandler):
81+ DEBOUNCE_SECONDS = settings.SMARTWATCH_DEBOUNCE_SECONDS
82+
83+ def __init__(self, *args, **kwargs):
84+ regexes = [".*.py", ".*.html", ".*/static/.*", ".*requirements.txt"]
85+ super().__init__(*args, **kwargs, regexes=regexes, ignore_directories=True)
86+ self.last_modified = time.time()
87+ self.changes_queue = ""
88+ self.is_restarting = False
89+ self.is_running = True
90+ """Adds all the changed files into a string to be processed at once"""
91+ prep_server()
92+ self.gunicorn = create_gunicorn()
93+ self.daphne = create_daphne()
94+ self.celery = create_celery()
95+
96+ def on_any_event(self, event):
97+ if event.event_type in ["modified", "created", "deleted", "moved"]:
98+ self.changes_queue += str(event.src_path)
99+ if not self.is_restarting:
100+ Thread(target=self.restart).start()
101+
102+ def restart(self):
103+ self.is_restarting = True
104+ prep_server()
105+ if ".py" or ".html" in self.changes_queue:
106+ self.gunicorn.send_signal(signal.SIGINT)
107+ self.daphne.send_signal(signal.SIGINT)
108+ self.celery.send_signal(signal.SIGINT)
109+ self.gunicorn.wait()
110+ self.gunicorn = create_gunicorn()
111+ self.daphne.wait()
112+ self.daphne = create_daphne()
113+ self.celery.wait()
114+ self.celery = create_celery()
115+ self.changes_queue = ""
116+ self.is_restarting = False
117+
118+ def stop(self):
119+ self.is_running = False
120+ self.gunicorn.send_signal(signal.SIGINT)
121+ self.daphne.send_signal(signal.SIGINT)
122+ self.celery.send_signal(signal.SIGINT)
123+
124+ def join(self):
125+ self.gunicorn.wait()
126+ self.daphne.wait()
127+ self.celery.wait()
128+
129+ def sleep(self, seconds):
130+ start = time.time()
131+ while self.is_running and time.time() - start < seconds:
132+ continue
133+
134+
135+def watch_server():
136+ server_handler = ServerHandler()
137+ observer = Observer()
138+ observer.schedule(server_handler, ".", recursive=True)
139+ observer.start()
140+
141+ def shutdown(*_):
142+ logging.warning("Stopping Smartwatch server from SIGKILL...")
143+ server_handler.stop()
144+ observer.stop()
145+ observer.join()
146+
147+ signal.signal(signal.SIGTERM, shutdown)
148+
149+ try:
150+ while observer.is_alive():
151+ observer.join(1)
152+ except KeyboardInterrupt:
153+ logging.warning("Stopping Smartwatch server...")
154+ finally:
155+ shutdown()