Upload 29 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+ugc_original_videos/0_16_07_500001604801190-yase.mp4 filter=lfs diff=lfs merge=lfs -text

data_processing/__init__.py

@@ -0,0 +1,2 @@
+# __init__.py
+# print("Data processing")

data_processing/split_train_test.py

@@ -0,0 +1,126 @@
+import pandas as pd
+import numpy as np
+import os
+import torch
+from sklearn.model_selection import train_test_split
+import logging
+
+#NR: original
+def process_lsvq(train_data_name, test_data_name, metadata_path, feature_path, network_name):
+    train_df = pd.read_csv(f'{metadata_path}/{train_data_name.upper()}_metadata.csv')
+    test_df = pd.read_csv(f'{metadata_path}/{test_data_name.upper()}_metadata.csv')
+
+    # grayscale videos, do not consider them for fair comparison
+    # grey_df_train = pd.read_csv(f'{metadata_path}/greyscale_report/{train_data_name.upper()}_greyscale_metadata.csv')
+    # grey_df_test = pd.read_csv(f'{metadata_path}/greyscale_report/{test_data_name.upper()}_greyscale_metadata.csv')
+    # grey_indices_train = grey_df_train.iloc[:, 0].tolist()
+    # grey_indices_test = grey_df_test.iloc[:, 0].tolist()
+    # train_df = train_df.drop(index=grey_indices_train).reset_index(drop=True)
+    # test_df = test_df.drop(index=grey_indices_test).reset_index(drop=True)
+    test_vids = test_df['vid']
+
+    # mos scores
+    train_scores = train_df['mos'].tolist()
+    test_scores = test_df['mos'].tolist()
+    train_mos_list = train_scores
+    test_mos_list = test_scores
+
+    # reorder columns
+    sorted_train_df = pd.DataFrame({'vid': train_df['vid'],  'framerate': train_df['framerate'], 'MOS': train_mos_list, 'MOS_raw': train_df['mos']})
+    sorted_test_df = pd.DataFrame({'vid': test_df['vid'], 'framerate': test_df['framerate'], 'MOS': test_mos_list, 'MOS_raw': test_df['mos']})
+
+    # use indices from the train and test DataFrames to split features
+    train_features = torch.load(f'{feature_path}/{network_name}_{train_data_name}_features.pt')
+    print(f"loaded {train_data_name}: dimensions are {train_features.shape}")
+    test_features = torch.load(f'{feature_path}/{network_name}_{test_data_name}_features.pt')
+
+    # grayscale videos
+    # train_mask = torch.ones(train_features.size(0), dtype=torch.bool, device=train_features.device)
+    # test_mask = torch.ones(test_features.size(0), dtype=torch.bool, device=test_features.device)
+    # train_mask[grey_indices_train] = False
+    # test_mask[grey_indices_test] = False
+    # train_features = train_features[train_mask]
+    # test_features = test_features[test_mask]
+    print(len(train_features))
+    print(len(test_features))
+
+    # save the files
+    sorted_train_df.to_csv(f'{metadata_path}mos_files/{train_data_name}_MOS_train.csv', index=False)
+    sorted_test_df.to_csv(f'{metadata_path}mos_files/{train_data_name}_MOS_test.csv', index=False)
+    os.makedirs(os.path.join(feature_path, "split_train_test"), exist_ok=True)
+    torch.save(train_features, f'{feature_path}/split_train_test/{network_name}_{train_data_name}_train_features.pt')
+    torch.save(test_features, f'{feature_path}/split_train_test/{network_name}_{test_data_name}_test_features.pt')
+
+    return train_features, test_features, test_vids
+
+def process_other(data_name, test_size, random_state, metadata_path, feature_path, network_name):
+    metadata_name = f'{data_name.upper()}_metadata.csv'
+    # for test finevd mos dimensions
+    # if data_name == 'finevd':
+    #     metadata_name = f'{data_name.upper()}_MOS_temporal_metadata.csv'
+    #     print(metadata_name)
+
+    # load CSV data
+    df = pd.read_csv(f'{metadata_path}/{metadata_name}')
+    # if data_name == 'youtube_ugc':
+    #     # grayscale videos, do not consider them for fair comparison
+    #     grey_df = pd.read_csv(f'{metadata_path}/greyscale_report/{data_name.upper()}_greyscale_metadata.csv')
+    #     grey_indices = grey_df.iloc[:, 0].tolist()
+    #     df = df.drop(index=grey_indices).reset_index(drop=True)
+
+    # get unique vids
+    unique_vids = df['vid'].unique()
+
+    # split videonames into train and test sets
+    train_vids, test_vids = train_test_split(unique_vids, test_size=test_size, random_state=random_state)
+
+    # split all_dfs into train and test based on vids
+    train_df = df[df['vid'].isin(train_vids)]
+    test_df = df[df['vid'].isin(test_vids)]
+
+    # mos scores
+    train_scores = train_df['mos'].tolist()
+    test_scores = test_df['mos'].tolist()
+    train_mos_list = train_scores
+    test_mos_list = test_scores
+
+    # reorder columns
+    sorted_train_df = pd.DataFrame({'vid': train_df['vid'],  'framerate': train_df['framerate'], 'MOS': train_mos_list, 'MOS_raw': train_df['mos']})
+    sorted_test_df = pd.DataFrame({'vid': test_df['vid'], 'framerate': test_df['framerate'], 'MOS': test_mos_list, 'MOS_raw': test_df['mos']})
+
+    # use indices from the train and test DataFrames to split features
+    features = torch.load(f'{feature_path}/{network_name}_{data_name}_features.pt')
+    # if data_name == 'youtube_ugc':
+    #     # features = np.delete(features, grey_indices, axis=0)
+    #     mask = torch.ones(features.size(0), dtype=torch.bool, device=features.device)
+    #     mask[grey_indices] = False
+    #     features = features[mask]
+    train_features = features[train_df.index]
+    test_features = features[test_df.index]
+
+    # save the files
+    sorted_train_df.to_csv(f'{metadata_path}mos_files/{data_name}_MOS_train.csv', index=False)
+    sorted_test_df.to_csv(f'{metadata_path}mos_files/{data_name}_MOS_test.csv', index=False)
+    os.makedirs(os.path.join(feature_path, "split_train_test"), exist_ok=True)
+    torch.save(train_features, f'{feature_path}/split_train_test/{network_name}_{data_name}_train_features.pt')
+    torch.save(test_features, f'{feature_path}/split_train_test/{network_name}_{data_name}_test_features.pt')
+
+    return train_features, test_features, test_vids
+
+
+if __name__ == '__main__':
+    network_name = "slowfast"
+    data_name = "test"
+    metadata_path = '../../metadata/'
+    feature_path = '../../features/konvid_1k_test/slowfast/'
+
+    # train test split
+    test_size = 0.2
+    random_state = None
+
+    if data_name == 'lsvq_train':
+        test_data_name = 'lsvq_test'
+        process_lsvq(data_name, test_data_name, metadata_path, feature_path, network_name)
+
+    else:
+        process_other(data_name, test_size, random_state, metadata_path, feature_path, network_name)

extractor/__init__.py

@@ -0,0 +1,2 @@
+# __init__.py
+# print("Initializing extractor")

extractor/extract_clip_embeds.py

@@ -0,0 +1,166 @@
+import re
+import torch
+import clip
+import numpy as np
+from numpy.linalg import norm
+from PIL import Image
+
+def get_quality_hint_from_metadata(mos, width, height, bitrate, bitdepth, framerate, quality_hints):
+    hint = []
+    if mos > 5:
+        mos = (mos / 100) * 5
+    if mos >= 4.5:
+        hint.append(quality_hints["mos"]["excellent"])
+    elif 3.5 <= mos < 4.5:
+        hint.append(quality_hints["mos"]["good"])
+    elif 2.5 <= mos < 3.5:
+        hint.append(quality_hints["mos"]["fair"])
+    elif 1.5 <= mos < 2.5:
+        hint.append(quality_hints["mos"]["bad"])
+    else:
+        hint.append(quality_hints["mos"]["poor"])
+
+    res = width * height
+    if res < 640 * 480:
+        hint.append(quality_hints["resolution"]["low"])
+    elif res < 1280 * 720:
+        hint.append(quality_hints["resolution"]["sd"])
+    else:
+        hint.append(quality_hints["resolution"]["hd"])
+    if bitrate < 500_000:
+        hint.append(quality_hints["bitrate"]["low"])
+    elif bitrate < 1_000_000:
+        hint.append(quality_hints["bitrate"]["medium"])
+    else:
+        hint.append(quality_hints["bitrate"]["high"])
+
+    if 0 < bitdepth <= 8:
+        hint.append(quality_hints["bitdepth"]["low"])
+    elif bitdepth == 0:
+        hint.append(quality_hints["bitdepth"]["standard"])
+    else:
+        hint.append(quality_hints["bitdepth"]["high"])
+    if framerate < 24:
+        hint.append(quality_hints["framerate"]["low"])
+    elif framerate > 60:
+        hint.append(quality_hints["framerate"]["high"])
+    else:
+        hint.append(quality_hints["framerate"]["standard"])
+    return " ".join(hint)
+
+def generate_caption(blip_processor, blip_model, device, image, prompt):
+    inputs = blip_processor(image, prompt, return_tensors="pt").to(device)
+    generated_ids = blip_model.generate(**inputs, max_new_tokens=50)
+    caption = blip_processor.decode(generated_ids[0], skip_special_tokens=True)
+    return caption
+
+def tensor_to_pil(image_tensor):
+    if isinstance(image_tensor, torch.Tensor):
+        arr = image_tensor.cpu().numpy()
+        if arr.ndim == 4 and arr.shape[0] == 1:
+            arr = arr[0]  # remove batch dimension
+        arr = arr.astype('uint8')
+    return Image.fromarray(arr)
+
+def extract_semantic_captions(blip_processor, blip_model, curr_frame, frag_residual, frag_frame, prompts, device, metadata=None, use_metadata_prompt=False):
+    quality_prompt_base = prompts["quality_prompt_base"]
+    residual_prompt = prompts["residual_prompt"]
+    frag_prompt = prompts["frag_prompt"]
+
+    quality_hint = ""
+    if use_metadata_prompt and metadata:
+        mos, width, height, bitrate, bitdepth, framerate = metadata
+        quality_hint = get_quality_hint_from_metadata(mos, width, height, bitrate, bitdepth, framerate, quality_hints=prompts["quality_hints"])
+
+    prompt_hints = []
+    if quality_hint:
+        prompt_hints.append(quality_hint)
+
+    quality_prompt = "\n\n".join(prompt_hints + [quality_prompt_base])
+    fragment_prompt = "\n\n".join(prompt_hints)
+    # print('content_prompt:', content_prompt)
+    # print('quality_prompt:', quality_prompt)
+    # print('residual_prompt:', fragment_prompt + "\n\n" + residual_prompt)
+    # print('frame_fragment_prompt:', fragment_prompt + "\n\n" + frag_prompt)
+
+    captions = {
+        "curr_frame_quality": generate_caption(blip_processor, blip_model, device, curr_frame, prompt=quality_prompt),
+        "frag_residual": generate_caption(blip_processor, blip_model, device, frag_residual, prompt=(fragment_prompt + "\n\n" + residual_prompt)),
+        "frag_frame": generate_caption(blip_processor, blip_model, device, frag_frame, prompt=(fragment_prompt + "\n\n" + frag_prompt))
+    }
+    return captions
+
+def clean_caption_text(text):
+    text = re.sub(r"- .*?stock videos & royalty-free footage", "", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+def dedup_keywords(text, split_tokens=[",", ".", ";"]):
+    for token in split_tokens:
+        text = text.replace(token, ",")
+    parts = [p.strip().lower() for p in text.split(",") if p.strip()]
+    seen = set()
+    unique_parts = []
+    for part in parts:
+        if part not in seen:
+            unique_parts.append(part)
+            seen.add(part)
+    return " ".join(unique_parts)  # good for embedding
+
+def get_clip_text_embedding(clip_model, device, text):
+    text_tokens = clip.tokenize([text]).to(device)
+    with torch.no_grad():
+        with torch.amp.autocast(device_type='cuda'):
+            text_features = clip_model.encode_text(text_tokens)
+            text_features = text_features / text_features.norm(dim=-1, keepdim=True)
+    return text_features.squeeze()
+
+def get_clip_image_embedding(clip_model, clip_preprocess, device, image):
+    image_input = clip_preprocess(image).unsqueeze(0).to(device)
+    with torch.no_grad():
+        with torch.amp.autocast(device_type='cuda'):
+            image_features = clip_model.encode_image(image_input)
+            image_features = image_features / image_features.norm(dim=-1, keepdim=True)
+    return image_features.squeeze()
+
+def extract_semantic_embeddings(clip_model, clip_preprocess, device, curr_frame, captions):
+    if not isinstance(curr_frame, Image.Image):
+        curr_frame = Image.fromarray(curr_frame)
+
+    quality_caption = dedup_keywords(clean_caption_text(captions["curr_frame_quality"]))
+    artifact_caption_1 = dedup_keywords(clean_caption_text(captions["frag_residual"]))
+    artifact_caption_2 = dedup_keywords(clean_caption_text(captions["frag_frame"]))
+    artifact_caption = dedup_keywords(f"{artifact_caption_1}, {artifact_caption_2}")
+
+    image_embed = get_clip_image_embedding(clip_model, clip_preprocess, device, curr_frame)
+    quality_embed = get_clip_text_embedding(clip_model, device, quality_caption)
+    artifact_embed = get_clip_text_embedding(clip_model, device, artifact_caption)
+    return image_embed, quality_embed, artifact_embed
+
+def extract_features_clip_embed(frames_info, metadata, clip_model, clip_preprocess, blip_processor, blip_model, prompts, device):
+    feature_image_embed = []
+    feature_quality_embed = []
+    feature_artifact_embed = []
+    for i, (curr_frame, frag_residual, frag_frame) in enumerate(frames_info):
+        curr_frame = tensor_to_pil(curr_frame)
+        frag_residual = tensor_to_pil(frag_residual)
+        frag_frame = tensor_to_pil(frag_frame)
+
+        captions = extract_semantic_captions(
+            blip_processor, blip_model,
+            curr_frame, frag_residual, frag_frame, prompts,
+            device,
+            metadata=metadata,
+            use_metadata_prompt=True,
+        )
+        image_embed, quality_embed, artifact_embed = extract_semantic_embeddings(clip_model, clip_preprocess, device, curr_frame, captions)
+        feature_image_embed.append(image_embed)
+        feature_quality_embed.append(quality_embed)
+        feature_artifact_embed.append(artifact_embed)
+
+    # concatenate features
+    image_embedding = torch.stack(feature_image_embed, dim=0)
+    quality_embedding = torch.stack(feature_quality_embed, dim=0)
+    artifact_embedding = torch.stack(feature_artifact_embed, dim=0)
+    # print("image_embedding.shape:", image_embedding.shape, "quality_embedding.shape:", quality_embedding.shape, "artifact_embedding.shape:", artifact_embedding.shape)
+    return image_embedding, quality_embedding, artifact_embedding

extractor/extract_frag.py

@@ -0,0 +1,327 @@
+import os
+import cv2
+import pandas as pd
+import time
+import torch
+import matplotlib.pyplot as plt
+import numpy as np
+from torchvision import transforms
+from torch.utils.data import DataLoader
+from PIL import Image
+from torch.utils import data
+
+
+class VideoDataset_feature(data.Dataset):
+    def __init__(self, filename_path, data_dir, database, transform, resize, patch_size=16, target_size=224, top_n=196):
+        super(VideoDataset_feature, self).__init__()
+        if isinstance(filename_path, str):
+            self.dataInfo = pd.read_csv(filename_path)
+        elif isinstance(filename_path, pd.DataFrame):
+            self.dataInfo = filename_path
+        else:
+            raise ValueError("filename_path: CSV file or DataFrame")
+        self.video_names = self.dataInfo['vid'].tolist()
+        self.videos_dir = data_dir
+        self.database = database
+        self.transform = transform
+        self.resize = resize
+        self.patch_size = patch_size
+        self.target_size = target_size
+        self.top_n = top_n
+        self.length = len(self.video_names)
+
+    def __len__(self):
+        return self.length
+
+    def __getitem__(self, idx):
+        if self.database in ['konvid_1k', 'test']:
+            video_clip_min = 8
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database == 'live_vqc':
+            video_clip_min = 10
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database == 'cvd_2014':
+            video_clip_min = 12
+            video_name = str(self.video_names[idx]) + '.avi'
+        elif self.database == 'youtube_ugc':
+            video_clip_min = 20
+            video_name = str(self.video_names[idx]) + '.mkv'
+        elif self.database == 'youtube_ugc_h264':
+            video_clip_min = 20
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database == 'live_yt_gaming':
+            video_clip_min = 7
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database == 'kvq':
+            video_clip_min = 8
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database in ['finevd', 'finevd_test']:
+            video_clip_min = 8
+            video_name = str(self.video_names[idx]) + '.mp4'
+        elif self.database in ['lsvq_test_1080p', 'lsvq_test', 'lsvq_train']:
+            video_clip_min = 8
+            video_name = str(self.video_names[idx]) + '.mp4'
+
+        # get metadata
+        row = self.dataInfo.iloc[idx]
+        if 'prediction_mode' in row and pd.notnull(row['prediction_mode']):
+            mos = float(row['prediction_mode'])
+        else:
+            mos = float(row['mos'])
+        metadata = (
+            mos,
+            int(row["width"]),
+            int(row["height"]),
+            int(row["bitrate"]),
+            int(row["bitdepth"]),
+            float(row["framerate"])
+        )
+        filename = os.path.join(self.videos_dir, video_name)
+
+        video_capture = cv2.VideoCapture(filename)
+        video_capture.set(cv2.CAP_PROP_BUFFERSIZE, 3)
+        if not video_capture.isOpened():
+            raise RuntimeError(f"Failed to open video: {filename}")
+
+        video_length = int(video_capture.get(cv2.CAP_PROP_FRAME_COUNT))
+        video_frame_rate = int(round(video_capture.get(cv2.CAP_PROP_FPS)))
+        fps = video_capture.get(cv2.CAP_PROP_FPS)
+        if fps is None or fps <= 1:
+            print(f"Invalid FPS={fps} for video {filename}. Using default")
+            fps = 2.0
+        frame_step = int(fps // 2)
+        video_clip = int(video_length / video_frame_rate) if video_frame_rate != 0 else 10
+        video_channel = 3
+        video_length_clip = 32
+
+        all_frame_tensor = torch.zeros((video_length, video_channel, self.resize, self.resize), dtype=torch.float32)
+        all_residual_frag_tensor = torch.zeros((video_length - 1, video_channel, self.resize, self.resize), dtype=torch.float32)
+        all_frame_frag_tensor = torch.zeros((video_length - 1, video_channel, self.resize, self.resize), dtype=torch.float32)
+
+        video_read_index = 0
+        frames_info = []
+        prev_frame = None
+        for i in range(video_length):
+            has_frames, frame = video_capture.read()
+            if has_frames:
+                curr_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                # frame features
+                curr_frame_tensor = self.transform(Image.fromarray(curr_frame))
+                all_frame_tensor[video_read_index] = curr_frame_tensor
+
+                # frame frag features
+                if prev_frame is not None:
+                    residual = cv2.absdiff(curr_frame, prev_frame)
+                    diff = self.get_patch_diff(residual)
+                    # frame residual fragment
+                    imp_patches, positions = self.extract_important_patches(residual, diff)
+                    imp_patches_pil = Image.fromarray(imp_patches.astype('uint8'))
+                    residual_frag_tensor = self.transform(imp_patches_pil)
+                    all_residual_frag_tensor[video_read_index] = residual_frag_tensor
+
+                    # current frame fragment
+                    ori_patches = self.get_original_frame_patches(curr_frame, positions)
+                    ori_patches_pil = Image.fromarray(ori_patches.astype('uint8'))
+                    frame_frag_tensor = self.transform(ori_patches_pil)
+                    all_frame_frag_tensor[video_read_index] = frame_frag_tensor
+
+                    # as a tuple
+                    if i % frame_step == 0:
+                        frames_info.append((curr_frame, imp_patches, ori_patches))
+                    video_read_index += 1
+            prev_frame = curr_frame
+        video_capture.release()
+
+        # Unfilled frames
+        self.fill_tensor(all_frame_tensor, video_read_index, video_length)
+        self.fill_tensor(all_residual_frag_tensor, video_read_index, video_length - 1)
+        self.fill_tensor(all_frame_frag_tensor, video_read_index, video_length - 1)
+
+        video_all = []
+        video_res_frag_all = []
+        video_frag_all = []
+        for i in range(video_clip):
+            clip_tensor = torch.zeros([video_length_clip, video_channel, self.resize, self.resize])
+            clip_res_frag_tensor = torch.zeros([video_length_clip, video_channel, self.resize, self.resize])
+            clip_frag_tensor = torch.zeros([video_length_clip, video_channel, self.resize, self.resize])
+
+            start_idx = i * video_frame_rate
+            end_idx = start_idx + video_length_clip
+            # frame features
+            if end_idx <= video_length:
+                clip_tensor = all_frame_tensor[start_idx:end_idx]
+            else:
+                clip_tensor[:(video_length - start_idx)] = all_frame_tensor[start_idx:]
+                clip_tensor[(video_length - start_idx):video_length_clip] = clip_tensor[video_length - start_idx - 1]
+
+            # frame frag features
+            if end_idx <= (video_length - 1):
+                clip_res_frag_tensor = all_residual_frag_tensor[start_idx:end_idx]
+                clip_frag_tensor = all_frame_frag_tensor[start_idx:end_idx]
+            else:
+                clip_res_frag_tensor[:(video_length - 1 - start_idx)] = all_residual_frag_tensor[start_idx:]
+                clip_frag_tensor[:(video_length - 1 - start_idx)] = all_frame_frag_tensor[start_idx:]
+                clip_res_frag_tensor[(video_length - 1 - start_idx):video_length_clip] = clip_res_frag_tensor[video_length - 1 - start_idx - 1]
+                clip_frag_tensor[(video_length - 1 - start_idx):video_length_clip] = clip_frag_tensor[video_length - 1 - start_idx - 1]
+
+            video_all.append(clip_tensor)
+            video_res_frag_all.append(clip_res_frag_tensor)
+            video_frag_all.append(clip_frag_tensor)
+
+        # Underfilling of clips
+        if video_clip < video_clip_min:
+            for i in range(video_clip, video_clip_min):
+                video_all.append(video_all[video_clip - 1])
+                video_res_frag_all.append(video_res_frag_all[video_clip - 1])
+                video_frag_all.append(video_frag_all[video_clip - 1])
+        return video_all, video_res_frag_all, video_frag_all, video_name, frames_info, metadata
+
+    @staticmethod
+    # duplicat the final frames
+    def fill_tensor(tensor, read_index, length):
+        if read_index < length:
+            tensor[read_index:length] = tensor[read_index - 1]
+
+    def get_patch_diff(self, residual_frame):
+        h, w = residual_frame.shape[:2]
+        patch_size = self.patch_size
+        h_adj = (h // patch_size) * patch_size
+        w_adj = (w // patch_size) * patch_size
+        residual_frame_adj = residual_frame[:h_adj, :w_adj]
+        # calculate absolute patch difference
+        diff = np.zeros((h_adj // patch_size, w_adj // patch_size))
+        for i in range(0, h_adj, patch_size):
+            for j in range(0, w_adj, patch_size):
+                patch = residual_frame_adj[i:i+patch_size, j:j+patch_size]
+                # absolute sum
+                diff[i // patch_size, j // patch_size] = np.sum(np.abs(patch))
+        return diff
+
+    def extract_important_patches(self, residual_frame, diff):
+        patch_size = self.patch_size
+        target_size = self.target_size
+        top_n = self.top_n
+
+        # find top n patches indices
+        patch_idx = np.unravel_index(np.argsort(-diff.ravel()), diff.shape)
+        top_patches = list(zip(patch_idx[0][:top_n], patch_idx[1][:top_n]))
+        sorted_idx = sorted(top_patches, key=lambda x: (x[0], x[1]))
+
+        imp_patches_img = np.zeros((target_size, target_size, residual_frame.shape[2]), dtype=residual_frame.dtype)
+        patches_per_row = target_size // patch_size  # 14
+        # order the patch in the original location relation
+        positions = []
+        for idx, (y, x) in enumerate(sorted_idx):
+            patch = residual_frame[y * patch_size:(y + 1) * patch_size, x * patch_size:(x + 1) * patch_size]
+            # new patch location
+            row_idx = idx // patches_per_row
+            col_idx = idx % patches_per_row
+            start_y = row_idx * patch_size
+            start_x = col_idx * patch_size
+            imp_patches_img[start_y:start_y + patch_size, start_x:start_x + patch_size] = patch
+            positions.append((y, x))
+        return imp_patches_img, positions
+
+    def get_original_frame_patches(self, original_frame, positions):
+        patch_size = self.patch_size
+        target_size = self.target_size
+        imp_original_patches_img = np.zeros((target_size, target_size, original_frame.shape[2]), dtype=original_frame.dtype)
+        patches_per_row = target_size // patch_size
+
+        for idx, (y, x) in enumerate(positions):
+            start_y = y * patch_size
+            start_x = x * patch_size
+            end_y = start_y + patch_size
+            end_x = start_x + patch_size
+
+            patch = original_frame[start_y:end_y, start_x:end_x]
+            row_idx = idx // patches_per_row
+            col_idx = idx % patches_per_row
+            target_start_y = row_idx * patch_size
+            target_start_x = col_idx * patch_size
+
+            imp_original_patches_img[target_start_y:target_start_y + patch_size,
+                                     target_start_x:target_start_x + patch_size] = patch
+        return imp_original_patches_img
+
+def visualise_tensor(tensors, num_frames_to_visualise=5, img_title='Frag'):
+    np_feat = tensors.numpy()
+    fig, axes = plt.subplots(1, num_frames_to_visualise, figsize=(15, 5))
+    for i in range(num_frames_to_visualise):
+        # move channels to last dimension for visualisation: (height, width, channels)
+        frame = np_feat[i].transpose(1, 2, 0)
+        # normalize to [0, 1] for visualisation
+        frame = (frame - frame.min()) / (frame.max() - frame.min())
+        axes[i].imshow(frame)
+        axes[i].axis('off')
+        axes[i].set_title(f'{img_title} {i + 1}')
+
+    plt.tight_layout()
+    plt.show()
+
+def visualise_image(frame, img_title='Residual Fragment', debug=True):
+    if debug:
+        plt.figure(figsize=(5, 5))
+        plt.imshow(frame)
+        plt.axis('off')
+        plt.title(img_title)
+        plt.show()
+
+if __name__ == "__main__":
+    database = 'test'
+    videos_dir = '../../test_videos/'
+    metadata_csv = '../../metadata/TEST_metadata.csv'
+    resize = 224
+    patch_size = 16
+    target_size = 224
+    top_n = 14 * 14
+    start_time = time.time()
+    resize_transform = transforms.Compose([
+        transforms.Resize([resize, resize]),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.45, 0.45, 0.45], std=[0.225, 0.225, 0.225])
+    ])
+
+    dataset = VideoDataset_feature(
+        filename_path=metadata_csv,
+        data_dir=videos_dir,
+        database=database,
+        transform=resize_transform,
+        resize=resize,
+        patch_size=patch_size,
+        target_size=target_size,
+        top_n=top_n
+    )
+
+    # test
+    dataloader = DataLoader(dataset, batch_size=1, shuffle=False)
+
+    start_time = time.time()
+    index = 0
+    video_segments, video_res_frag_all, video_frag_all, video_name, frames_info, metadata = dataset[index]
+
+    print(f"\n=== Video Processed ===")
+    print(f"Video Name: {video_name}")
+    print(f"Metadata: [MOS, width, height, bitrate, bitdepth, framerate] = {metadata}")
+    print(f"Number of Segments: {len(video_segments)}")
+    print(f"Number of Video Residual Fragment Segments: {len(video_res_frag_all)}")
+    print(f"Number of Video Fragment Segments: {len(video_frag_all)}")
+    print(f"Shape of Each Segment: {video_segments[0].shape}")  # (video_length_clip, channels, height, width)
+    print(f"Shape of Each Residual Fragment Segments: {video_res_frag_all[0].shape}")
+    print(f"Shape of Each Fragment Segments: {video_frag_all[0].shape}")
+    print(f"Total Key Frame Tuples (frames_info): {len(frames_info)}")
+    curr_frame, imp_patch, ori_patch = frames_info[0]
+    print("curr_frame shape:", np.array(curr_frame).shape)
+    print("imp_patch shape:", np.array(imp_patch).shape)
+    print("ori_patch shape:", np.array(ori_patch).shape)
+
+    # visualisation
+    curr_frame, frag_residual, frag_frame = frames_info[0]
+    visualise_image(curr_frame, 'Current Frame')
+    visualise_image(frag_residual, 'Residual Fragment')
+    visualise_image(frag_frame, 'Frame Fragment')
+
+    visualise_tensor(video_segments[0], num_frames_to_visualise=5, img_title='Frame')
+    visualise_tensor(video_res_frag_all[0], num_frames_to_visualise=5, img_title='Residual Frag')
+    visualise_tensor(video_frag_all[0], num_frames_to_visualise=5, img_title='Frame Frag')
+    print(f"\nTotal Time: {time.time() - start_time:.2f} seconds")

extractor/extract_slowfast_clip.py

@@ -0,0 +1,64 @@
+import torch
+import torch.nn as nn
+from pytorchvideo.models.hub import slowfast_r50
+
+def pack_pathway_output(frames, device):
+    fast_pathway = frames
+    # temporal sampling from the fast pathway.
+    slow_pathway = torch.index_select(
+        frames,
+        2,
+        torch.linspace(0, frames.shape[2] - 1, frames.shape[2] // 4).long(),
+    )
+    return [slow_pathway.to(device), fast_pathway.to(device)]
+
+
+class SlowFast(torch.nn.Module):
+    def __init__(self):
+        super(SlowFast, self).__init__()
+        slowfast_pretrained_features = nn.Sequential(*list(slowfast_r50(pretrained=True).children())[0])
+
+        self.feature_extraction = torch.nn.Sequential()
+        self.slow_avg_pool = torch.nn.Sequential()
+        self.fast_avg_pool = torch.nn.Sequential()
+        self.adp_avg_pool = torch.nn.Sequential()
+
+        for x in range(0, 5):
+            self.feature_extraction.add_module(str(x), slowfast_pretrained_features[x])
+
+        self.slow_avg_pool.add_module('slow_avg_pool', slowfast_pretrained_features[5].pool[0])
+        self.fast_avg_pool.add_module('fast_avg_pool', slowfast_pretrained_features[5].pool[1])
+        self.adp_avg_pool.add_module('adp_avg_pool', slowfast_pretrained_features[6].output_pool)
+
+    def forward(self, x):
+        with torch.no_grad():
+            x = self.feature_extraction(x)
+            slow_feature = self.slow_avg_pool(x[0])
+            fast_feature = self.fast_avg_pool(x[1])
+            slow_feature = self.adp_avg_pool(slow_feature)
+            fast_feature = self.adp_avg_pool(fast_feature)
+        return slow_feature, fast_feature
+
+def extract_features_slowfast_pool(video, model, device):
+    slow_features_list = []
+    fast_features_list = []
+
+    with torch.amp.autocast(device_type='cuda'):
+        for idx, segment in enumerate(video):
+            segment = segment.permute(0, 2, 1, 3, 4)
+            inputs = pack_pathway_output(segment, device)
+            # print(f"Inputs shapes: slow={inputs[0].shape}, fast={inputs[1].shape}")
+
+            # extract features
+            slow_feature, fast_feature = model(inputs)
+            # global average pooling to reduce dimensions
+            slow_feature = slow_feature.mean(dim=[2, 3, 4])  # Pool over spatial and temporal dims
+            fast_feature = fast_feature.mean(dim=[2, 3, 4])
+            slow_features_list.append(slow_feature)
+            fast_features_list.append(fast_feature)
+
+    # concatenate pooled features
+    slow_features = torch.cat(slow_features_list, dim=0)
+    fast_features = torch.cat(fast_features_list, dim=0)
+    slowfast_features = torch.cat((slow_features, fast_features), dim=1)  # along feature dimension
+    return slow_features, fast_features, slowfast_features

extractor/extract_swint_clip.py

@@ -0,0 +1,40 @@
+import torch
+import torch.nn as nn
+from timm import create_model
+
+class Identity(nn.Module):
+    def __init__(self):
+        super(Identity, self).__init__()
+
+    def forward(self, x):
+        return x
+
+class SwinT(nn.Module):
+    def __init__(self, model_name='swin_base_patch4_window7_224', global_pool='avg', pretrained=True):
+        super(SwinT, self).__init__()
+        self.swin_model = create_model(
+            model_name, pretrained=pretrained, global_pool=global_pool
+        )
+        self.swin_model.head = Identity()  # Remove classification head
+        self.global_pool = global_pool
+
+    def forward(self, x):
+        features = self.swin_model(x)  # Shape: (batch_size, 7, 7, 1024)
+        if self.global_pool == 'avg':
+            features = features.mean(dim=[1, 2])  # Global pool
+        return features
+
+def extract_features_swint_pool(video, model, device):
+    swint_feature_list = []
+
+    with torch.amp.autocast(device_type='cuda'):
+        for segment in video:
+            # Flatten the segment into a batch of frames
+            frames = segment.squeeze(0).to(device)  # Shape: (32, 3, 224, 224)
+
+            swint_features = model(frames)  # Shape: (32, feature_dim)
+            swint_feature_list.append(swint_features)
+
+        # Concatenate features across segments
+        features = torch.cat(swint_feature_list, dim=0)  # Shape: (num_frames, feature_dim)
+    return features

model/finetune/cvd_2014_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:577a57259bcb14d536cb9be01e735a9659a418523a5f6882e1d6f0aa34a1a879
+size 13511298

model/finetune/finevd_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96f2baf34f0aeb853cdaf498a38b0bfaa4d05085ee7ad0bbe083050ed7613b9c
+size 13511148

model/finetune/konvid_1k_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c96602e5c2aeb644487af17215f3dd79c48f56f06c7a41a130a48cb70f7de4ee
+size 13511794

model/finetune/kvq_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76b2e72b802565a29892f2570555575ee00982253591f914e78528118a32a7a7
+size 13511103

model/finetune/live_vqc_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0962ea94657840d3afe4b7099ae532635c7cf1e21dab3c9df476d090fc892cb1
+size 13511298

model/finetune/live_yt_gaming_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9030af7485bbda2661c8c208c288f0b50cead0bc1bf600934eb02dd2e1893796
+size 13511874

model/finetune/youtube_ugc_h264_camp-vqa_fine_tuned_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a19c4fb90d6b44929f0dcfb276d5daf44b12f73b08a8b861cd8fa93fcdd6924
+size 13511418

model/wo_finetune/cvd_2014_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e30fbbbc821527e4204d84b1145f9ace84061779208a313180abd2cb8b377368
+size 13506412

model/wo_finetune/finevd_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c47579161d9bf8037d12baebcfc2d4ca49d0037d3dc77612d199606d1f43fe48
+size 13506031

model/wo_finetune/konvid_1k_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d73eb32a787b243a7b5aacbc8bf16ea91179f6aa89dc6980081ff422a2c05a9
+size 13506487

model/wo_finetune/kvq_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ca9718e13e7446df65bfdfb519c63004eb5df85c19260fa37e7f05f068bdd52
+size 13505998

model/wo_finetune/live_vqc_camp-vqa_Mlp_byrmse_trained_mode.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:464a2fa7f837658c01f113b503f37699cf2e9e809e258f09a0dc08808e5baf88
+size 13506412

model/wo_finetune/live_yt_gaming_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af0893a05ebc27512eede807f8c663f962b1fe6578aeb9d597abeefab58e304b
+size 13506271

model/wo_finetune/lsvq_train_camp-vqa_Mlp_byrmse_trained_model_1080p.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3beef20985f987dc89b33524f55d136ab23e2aab6eb936d29c3d2cc7a3d14ae
+size 13512098

model/wo_finetune/lsvq_train_camp-vqa_Mlp_byrmse_trained_model_kfold.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bec5863ae799a9f496d597d576b7e37dcdbea5df9e1c05a80d5da66591972fc7
+size 13512754

model/wo_finetune/youtube_ugc_h264_camp-vqa_Mlp_byrmse_trained_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8efe4d8b5817453da1dd6664914e5d0b5a307b7f5746b17939d7059622efcbdd
+size 13506692

ugc_original_videos/0_16_07_500001604801190-yase.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41b1f2a7cf48cda0a3adc956a98c864115dc83b692fcfb26d46814cb9fd9e9bc
+size 3883352