Merge pull request #1057 from AnkurPrabhu/new-download-fix

new-download-fix

api/all_tasks.py

@@ -0,0 +1,90 @@
+import io
+import os
+import uuid
+import zipfile
+from datetime import datetime
+
+import pytz
+from django.conf import settings
+from django.utils import timezone
+from django_q.tasks import AsyncTask, schedule
+
+import api.util as util
+from api.models.long_running_job import LongRunningJob
+
+
+def create_download_job(job_type, user, photos, filename):
+    job_id = uuid.uuid4()
+    lrj = LongRunningJob.objects.create(
+        started_by=user,
+        job_id=job_id,
+        queued_at=datetime.now().replace(tzinfo=pytz.utc),
+        job_type=job_type,
+    )
+    if job_type == LongRunningJob.JOB_DOWNLOAD_PHOTOS:
+        AsyncTask(
+            zip_photos_task, job_id=job_id, user=user, photos=photos, filename=filename
+        ).run()
+
+    lrj.save()
+    return job_id
+
+
+def zip_photos_task(job_id, user, photos, filename):
+    lrj = LongRunningJob.objects.get(job_id=job_id)
+    lrj.started_at = datetime.now().replace(tzinfo=pytz.utc)
+    count = len(photos)
+    lrj.result = {"progress": {"current": 0, "target": count}}
+    lrj.save()
+    output_directory = os.path.join(settings.MEDIA_ROOT, "zip")
+    zip_file_name = filename
+    done_count = 0
+    try:
+        if not os.path.exists(output_directory):
+            os.mkdir(output_directory)
+        mf = io.BytesIO()
+        photos_name = {}
+
+        for photo in photos.values():
+            done_count = done_count + 1
+            photo_name = os.path.basename(photo.main_file.path)
+            if photo_name in photos_name:
+                photos_name[photo_name] = photos_name[photo_name] + 1
+                photo_name = str(photos_name[photo_name]) + "-" + photo_name
+            else:
+                photos_name[photo_name] = 1
+            with zipfile.ZipFile(mf, mode="a", compression=zipfile.ZIP_DEFLATED) as zf:
+                zf.write(photo.main_file.path, arcname=photo_name)
+            lrj.result = {"progress": {"current": done_count, "target": count}}
+            lrj.save()
+        with open(os.path.join(output_directory, zip_file_name), "wb") as output_file:
+            output_file.write(mf.getvalue())
+
+    except Exception as e:
+        util.logger.error("Error while converting files to zip: {}".format(e))
+
+    lrj.finished_at = datetime.now().replace(tzinfo=pytz.utc)
+    lrj.finished = True
+    lrj.save()
+    # scheduling a task to delete the zip file after a day
+    execution_time = timezone.now() + timezone.timedelta(days=1)
+    schedule("api.all_tasks.delete_zip_file", filename, next_run=execution_time)
+    return os.path.join(output_directory, zip_file_name)
+
+
+def delete_zip_file(filename):
+    file_path = os.path.join(settings.MEDIA_ROOT, "zip", filename)
+    try:
+        if not os.path.exists(file_path):
+            util.logger.error(
+                "Error while deleting file not found at : {}".format(file_path)
+            )
+            return
+        else:
+            os.remove(file_path)
+            util.logger.info("file deleted sucessfully at path : {}".format(file_path))
+            return
+
+    except Exception as e:
+        util.logger.error("Error while deleting file: {}".format(e))
+        return e

api/batch_jobs.py

@@ -9,12 +9,11 @@
 
 import api.util as util
 from api.image_similarity import build_image_similarity_index
+from api.ml_models import download_models
 from api.models.long_running_job import LongRunningJob
 from api.models.photo import Photo
 from api.semantic_search.semantic_search import semantic_search_instance
 
-from api.ml_models import download_models
-
 
 def create_batch_job(job_type, user):
     job_id = uuid.uuid4()

api/face_recognition.py

@@ -1,5 +1,5 @@
-import requests
 import numpy as np
+import requests
 
 
 def get_face_encodings(image_path, known_face_locations):

api/geocode/geocode.py

@@ -2,6 +2,7 @@
 from constance import config as site_config
 
 import api.util as util
+
 from .config import get_provider_config, get_provider_parser
 
 

api/im2txt/build_vocab.py

@@ -32,8 +32,8 @@ def __len__(self):
 
 
 def build_vocab(json, threshold):
-    from pycocotools.coco import COCO
     import nltk
+    from pycocotools.coco import COCO
 
     """Build a simple vocabulary wrapper."""
     coco = COCO(json)

api/im2txt/data_loader.py

@@ -1,8 +1,8 @@
 import os
-from PIL import Image
 
 import torch
 import torch.utils.data as data
+from PIL import Image
 
 
 class CocoDataset(data.Dataset):

api/im2txt/model.py

@@ -1,7 +1,6 @@
 import torch
 import torch.nn as nn
 import torchvision.models as models
-from torch.nn.utils.rnn import pack_padded_sequence
 
 
 class EncoderCNN(nn.Module):

api/im2txt/sample.py

@@ -1,15 +1,14 @@
 import os
 import pickle
 
+import onnxruntime as ort
 import torch
 from django.conf import settings
+from numpy import asarray
 from PIL import Image
 from torchvision import transforms
 
 from api.im2txt.model import DecoderRNN, EncoderCNN
-import onnxruntime as ort
-
-from numpy import asarray
 
 embed_size = 256
 hidden_size = 512
@@ -162,7 +161,7 @@ def generate_caption(
         return sentence
 
     def export_onnx(self, encoder_output_path, decoder_output_path):
-        from torch.onnx import export, dynamo_export
+        from torch.onnx import dynamo_export, export
 
         self.load_models()
 
@@ -216,10 +215,7 @@ def export_onnx(self, encoder_output_path, decoder_output_path):
             ".onnx", "_quantizedyn.onnx"
         )
 
-        from onnxruntime.quantization import (
-            quantize_dynamic,
-        )
-
+        from onnxruntime.quantization import quantize_dynamic
         from onnxruntime.quantization.shape_inference import quant_pre_process
 
         quant_pre_process_encoder_output_path = encoder_output_path.replace(

api/migrations/0055_alter_longrunningjob_job_type.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.6 on 2023-10-27 13:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("api", "0054_user_cluster_selection_epsilon_user_min_samples"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="longrunningjob",
+            name="job_type",
+            field=models.PositiveIntegerField(
+                choices=[
+                    (1, "Scan Photos"),
+                    (2, "Generate Event Albums"),
+                    (3, "Regenerate Event Titles"),
+                    (4, "Train Faces"),
+                    (5, "Delete Missing Photos"),
+                    (7, "Scan Faces"),
+                    (6, "Calculate Clip Embeddings"),
+                    (8, "Find Similar Faces"),
+                    (9, "Download Selected Photos"),
+                ]
+            ),
+        ),
+    ]

api/ml_models.py

@@ -1,13 +1,14 @@
-import pytz
+import tarfile
 from datetime import datetime
-from django.conf import settings
+from pathlib import Path
 
-from api.models.long_running_job import LongRunningJob
-import api.util as util
+import pytz
 import requests
-import tarfile
-from pathlib import Path
 from constance import config as site_config
+from django.conf import settings
+
+import api.util as util
+from api.models.long_running_job import LongRunningJob
 
 
 class MlTypes:

api/models/long_running_job.py

@@ -18,6 +18,7 @@ class LongRunningJob(models.Model):
     JOB_CALCULATE_CLIP_EMBEDDINGS = 6
     JOB_SCAN_FACES = 7
     JOB_CLUSTER_ALL_FACES = 8
+    JOB_DOWNLOAD_PHOTOS = 9
     JOB_DOWNLOAD_MODELS = 10
 
     JOB_TYPES = (
@@ -29,6 +30,7 @@ class LongRunningJob(models.Model):
         (JOB_SCAN_FACES, "Scan Faces"),
         (JOB_CALCULATE_CLIP_EMBEDDINGS, "Calculate Clip Embeddings"),
         (JOB_CLUSTER_ALL_FACES, "Find Similar Faces"),
+        (JOB_DOWNLOAD_PHOTOS, "Download Selected Photos"),
         (JOB_DOWNLOAD_MODELS, "Download Models"),
     )
 

api/models/photo.py

@@ -11,15 +11,15 @@
 from django.db import models
 from django.db.models import Q
 from django.db.utils import IntegrityError
-from api.im2txt.sample import Im2txt
-from api.face_recognition import get_face_encodings, get_face_locations
 
 import api.date_time_extractor as date_time_extractor
 import api.models
 import api.util as util
 from api.exif_tags import Tags
+from api.face_recognition import get_face_encodings, get_face_locations
 from api.geocode import GEOCODE_VERSION
 from api.geocode.geocode import reverse_geocode
+from api.im2txt.sample import Im2txt
 from api.models.file import File
 from api.models.user import User, get_deleted_user
 from api.places365.places365 import place365_instance

api/places365/wideresnet.py

@@ -1,7 +1,7 @@
 import math
 import os
-import torch.nn as nn
 
+import torch.nn as nn
 from django.conf import settings
 
 model_path = os.path.join(settings.MEDIA_ROOT, "data_models", "resnet18-5c106cde.pth")

api/serializers/user.py

@@ -6,10 +6,10 @@
 from rest_framework.exceptions import ValidationError
 
 from api.batch_jobs import create_batch_job
+from api.ml_models import do_all_models_exist
 from api.models import LongRunningJob, Photo, User
 from api.serializers.simple import PhotoSuperSimpleSerializer
 from api.util import logger
-from api.ml_models import do_all_models_exist
 
 
 class UserSerializer(serializers.ModelSerializer):

api/tests/test_only_photos_or_only_videos.py

@@ -1,34 +1,28 @@
-
-from django.utils import timezone
-from unittest.mock import patch
 from django.test import TestCase
+from django.utils import timezone
 from rest_framework.test import APIClient
-from api.models.album_date import AlbumDate
-from api.tests.utils import create_test_photos, create_test_user,create_test_photo
-
 
+from api.models.album_date import AlbumDate
+from api.tests.utils import create_test_photo, create_test_user
 
 
 class OnlyPhotosOrOnlyVideosTest(TestCase):
     def setUp(self):
         self.client = APIClient()
         self.user = create_test_user()
         self.client.force_authenticate(user=self.user)
-        
+
     def test_only_photos(self):
         now = timezone.now()
-        photo= create_test_photo( owner=self.user, added_on=now,public=True)
+        photo = create_test_photo(owner=self.user, added_on=now, public=True)
 
-        album=AlbumDate(owner=self.user)
-        album.id=1
+        album = AlbumDate(owner=self.user)
+        album.id = 1
         album.photos.add(photo)
         album.save()
 
         response = self.client.get("/api/albums/date/list?photo=true").url
-        response =self.client.get(response)
+        response = self.client.get(response)
 
         data = response.json()
         self.assertEqual(1, len(data["results"]))
-
-
-

api/tests/test_zip_list_photos_view_v2.py

@@ -0,0 +1,36 @@
+from unittest.mock import patch
+
+from django.test import TestCase
+from django.utils import timezone
+from rest_framework.test import APIClient
+
+from api.models.long_running_job import LongRunningJob
+from api.tests.utils import create_test_photos, create_test_user
+
+
+class PhotoListWithoutTimestampTest(TestCase):
+    def setUp(self):
+        self.client = APIClient()
+        self.user = create_test_user()
+        self.client.force_authenticate(user=self.user)
+
+    @patch("shutil.disk_usage")
+    def test_download(self, patched_shutil):
+        # test download function when we have enough storage
+        patched_shutil.return_value.free = 500000000
+        now = timezone.now()
+        create_test_photos(number_of_photos=1, owner=self.user, added_on=now, size=100)
+
+        response = self.client.get("/api/photos/notimestamp/")
+        img_hash = response.json()["results"][0]["url"]
+        datadict = {"owner": self.user, "image_hashes": [img_hash]}
+
+        response_2 = self.client.post("/api/photos/download", data=datadict)
+        lrr_job = LongRunningJob.objects.all()[0]
+        self.assertEqual(lrr_job.job_id, response_2.json()["job_id"])
+        self.assertEqual(response_2.status_code, 200)
+
+        # test download function when we dont have enough storage
+        patched_shutil.return_value.free = 0
+        response_3 = self.client.post("/api/photos/download", data=datadict)
+        self.assertEqual(response_3.status_code, 507)

api/util.py

@@ -3,8 +3,6 @@
 import os.path
 
 import exiftool
-import requests
-from constance import config as site_config
 from django.conf import settings
 
 logger = logging.getLogger("ownphotos")