on tracking now gives device erro on some videos, but those who dont, give good results

.gitignore

@@ -139,5 +139,6 @@ docs/api
 events.out.tfevents*
 pretrained
 YOLOX_outputs
+
 .idea/*
 

experiments/meta_gpu_resume_fake.sh

@@ -0,0 +1,20 @@
+#PBS -N meta_bytetrack_resume_fake
+#PBS -m abe
+#PBS -M [email protected]
+#PBS -l nodes=1:ppn=1:gpus=1
+#PBS -q DP
+#PBS -l mem=15GB
+
+export LD_LIBRARY_PATH=/share/apps/cuda/cuda-10.1/lib64:$LD_LIBRARY_PATH
+export PATH=/share/apps/cuda/cuda-10.1/bin/:$PATH
+
+
+source /share/apps/Anaconda/anaconda3.6/bin/activate abdollahpour.ce.sharif
+conda activate abd_env
+
+
+cd /home/abdollahpour.ce.sharif/ByteTrack
+
+
+
+python tools/train.py -t metamot -f exps/example/metamot/yolox_x_mot17_on_mot20.py -d 1 -b 1 --fp16 -o --resume --start_epoch 80 -c /home/abdollahpour.ce.sharif/ByteTrack/meta_experiments/train_17_on_20_resume2/best_ckpt.pth.tar --local_rank 0 -expn train_17_on_20_resume_fake

experiments/track_on_20_no_adapt.sh

@@ -0,0 +1,18 @@
+#PBS -N track_17_on_20_no_adapt
+#PBS -m abe
+#PBS -M [email protected]
+#PBS -l nodes=1:ppn=1:gpus=1
+#PBS -q cuda9
+
+
+export LD_LIBRARY_PATH=/share/apps/cuda/cuda-10.1/lib64:$LD_LIBRARY_PATH
+export PATH=/share/apps/cuda/cuda-10.1/bin/:$PATH
+source /share/apps/Anaconda/anaconda3.6/bin/activate abdollahpour.ce.sharif
+conda activate abd_env
+
+
+cd /home/abdollahpour.ce.sharif/ByteTrack
+
+
+
+python tools/track.py -t metamot -f exps/example/metamot/yolox_x_mot17_on_mot20.py -d 1 -b 1 -c /home/abdollahpour.ce.sharif/ByteTrack/meta_experiments/train_17_on_20_resume2/best_ckpt.pth.tar --local_rank 0 -expn track_17_on_20_no_adapt --mot20

experiments/track_on_20_with_gt.sh

@@ -0,0 +1,16 @@
+#PBS -N track_17_on_20_ada_4_with_GT
+#PBS -m abe
+#PBS -M [email protected]
+#PBS -l nodes=1:ppn=1:gpus=1
+#PBS -q cuda9
+
+
+export LD_LIBRARY_PATH=/share/apps/cuda/cuda-10.1/lib64:$LD_LIBRARY_PATH
+export PATH=/share/apps/cuda/cuda-10.1/bin/:$PATH
+source /share/apps/Anaconda/anaconda3.6/bin/activate abdollahpour.ce.sharif
+conda activate abd_env
+cd /home/abdollahpour.ce.sharif/ByteTrack
+
+
+
+python tools/track.py -t metamot -f exps/example/metamot/yolox_x_mot17_on_mot20.py -d 1 -b 1 -c /home/abdollahpour.ce.sharif/ByteTrack/meta_experiments/train_17_on_20_resume2/best_ckpt.pth.tar --local_rank 0 -expn track_17_on_20_ada_4_with_GT --mot20 --adaptation_period 4 --fp16 --use_existing_files

exps/example/metamot/yolox_x_mot17_on_mot20.py

@@ -32,12 +32,14 @@ class Exp(MyMetaExp):
         print('train_anns', self.train_anns)
         print('val_anns', self.val_anns)
         self.input_size = (800, 1440)
-        self.test_size = (896, 1600)
+        # TODO: try this
+        self.test_size = (800, 1440)
+        # self.test_size = (896, 1600)
         # self.test_size = (736, 1920)
         self.random_size = (20, 36)
         self.max_epoch = 80
         self.print_interval = 250
-        self.eval_interval = 5
+        self.eval_interval = 10
         self.test_conf = 0.001
         self.nmsthre = 0.7
         self.no_aug_epochs = 10
@@ -109,7 +111,7 @@ class Exp(MyMetaExp):
         return train_loaders
 
     def get_eval_loaders(self, batch_size, is_distributed, testdev=False):
-        from yolox.data import MOTDataset, ValTransform, ValTransformWithPseudo
+        from yolox.data import MOTDataset, ValTransform, ValTransformWithPseudo, TrainTransform
         val_loaders = []
         for val_ann in self.val_anns:
             valdataset = MOTDataset(
@@ -119,9 +121,10 @@ class Exp(MyMetaExp):
                 name='train',  # change to train when running on training set
                 preproc=ValTransformWithPseudo(
                     rgb_means=(0.485, 0.456, 0.406),
-                    std=(0.229, 0.224, 0.225),
+                    std=(0.229, 0.224, 0.225), max_labels=500,
                 ),
-                load_weak=True
+
+                load_weak=False
             )
 
             if is_distributed:
@@ -142,7 +145,7 @@ class Exp(MyMetaExp):
             val_loaders.append(val_loader)
         return val_loaders
 
-    def get_evaluator(self, batch_size, is_distributed, testdev=False):
+    def get_evaluators(self, batch_size, is_distributed, testdev=False):
         from yolox.evaluators import COCOEvaluator
 
         val_loaders = self.get_eval_loaders(batch_size, is_distributed, testdev=testdev)

tools/track.py

@@ -134,7 +134,7 @@ def compare_dataframes(gts, ts):
     return accs, names
 
 
-def process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder, val_ann):
+def process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder):
     file_name = os.path.join(exp.output_dir, args.experiment_name)
     rank = args.local_rank
     if rank == 0:
@@ -158,7 +158,7 @@ def process_loader(args, exp, val_loader, model, is_distributed, trt_file, decod
     # start evaluate
     *_, summary = evaluator.evaluate(
         model, is_distributed, args.fp16, trt_file, decoder, exp.test_size, results_folder,
-        adaptation_period=adaptation_period,
+        adaptation_period=adaptation_period, eval_det=False
     )
     logger.info("\n" + summary)
 
@@ -262,7 +262,7 @@ def load_model(args, exp, is_distributed):
         exp.test_size = (args.tsize, args.tsize)
 
     if args.task == "metamot":
-        model = l2l.algorithms.MAML(model, lr=exp.inner_lr, first_order=exp.first_order, allow_nograd=True)
+        model = l2l.algorithms.MAML(model, lr=exp.inner_lr, first_order=exp.first_order)
     torch.cuda.set_device(rank)
     model.cuda(rank)
     model.eval()
@@ -328,14 +328,14 @@ def main(exp, args, num_gpu):
     if args.task == 'metamot':
         val_loaders = exp.get_eval_loaders(args.batch_size, is_distributed, args.test)
         if not args.use_existing_files:
-            for val_loader, val_ann in zip(val_loaders, exp.val_anns):
+            for val_loader in val_loaders:
                 logger.info('processing loader...')
-                process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder, val_ann)
+                process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder)
         eval_MOT(args, exp)
     else:
         if not args.use_existing_files:
             val_loader = exp.get_eval_loader(args.batch_size, is_distributed, args.test)
-            process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder, exp.val_ann)
+            process_loader(args, exp, val_loader, model, is_distributed, trt_file, decoder)
         eval_MOT(args, exp, exp.val_ann)
 
 

yolox/core/meta_trainer.py

@@ -137,7 +137,12 @@ class MetaTrainer:
                 p.grad.data.mul_(1.0 / self.args.batch_size)
 
         self.optimizer.zero_grad()
-        self.scaler.scale(loss).backward()
+
+        logger.info("loss Norm: {} , scale {}".format(torch.norm(loss), self.scaler.get_scale()))
+        loss = self.scaler.scale(loss)
+        logger.info("loss Norm: {} , scale {}".format(torch.norm(loss), self.scaler.get_scale()))
+        # self.scaler.scale(loss).backward()
+        loss.backward()
         self.scaler.step(self.optimizer)
         self.scaler.update()
 

yolox/data/__init__.py

@@ -2,7 +2,7 @@
 # -*- coding:utf-8 -*-
 # Copyright (c) Megvii, Inc. and its affiliates.
 
-from .data_augment import TrainTransform, ValTransform,ValTransformWithPseudo
+from .data_augment import TrainTransform, ValTransform, ValTransformWithPseudo
 from .data_prefetcher import DataPrefetcher
 from .dataloading import DataLoader, get_yolox_datadir
 from .datasets import *

yolox/data/data_augment.py

@@ -317,7 +317,7 @@ class ValTransformWithPseudo:
         data
     """
 
-    def __init__(self, rgb_means=None, std=None, swap=(2, 0, 1), max_labels=100):
+    def __init__(self, rgb_means=None, std=None, swap=(2, 0, 1), max_labels=150):
         self.means = rgb_means
         self.swap = swap
         self.std = std
@@ -330,6 +330,7 @@ class ValTransformWithPseudo:
         labels = targets[:, 4].copy()
         ids = targets[:, 5].copy()
         if len(boxes) == 0:
+            print('inside if 1')
             targets = np.zeros((self.max_labels, 6), dtype=np.float32)
             image, r_o = preproc(image, input_dim, self.means, self.std)
             image = np.ascontiguousarray(image, dtype=np.float32)
@@ -359,6 +360,7 @@ class ValTransformWithPseudo:
         ids_t = ids[mask_b]
 
         if len(boxes_t) == 0:
+            print('inside if 2')
             image_t, r_o = preproc(image_o, input_dim, self.means, self.std)
             boxes_o *= r_o
             boxes_t = boxes_o

yolox/data/datasets/mot.py

@@ -6,6 +6,7 @@ import os
 
 from ..dataloading import get_yolox_datadir
 from .datasets_wrapper import Dataset
+import sys
 
 
 class MOTDataset(Dataset):
@@ -102,25 +103,30 @@ class MOTDataset(Dataset):
         )
         head_tail = os.path.split(img_file)
         # label_path = os.path.join(head_tail[0],   head_tail[1].replace('.jpg','.txt'))
-
+        # sys.stderr.write('original shape' + str(res.shape) + '\n values \n' + str(res))
         if self.load_weak:
             weak_label_path = os.path.join(head_tail[0], head_tail[1] + '_weak_yolox-x.txt')
             # load weak labels from weak_label_path
             width = img_info[1]
             height = img_info[0]
-            labels = np.loadtxt(weak_label_path)
-            res = np.ones_like(labels)
-            labels[2, :] *= width
-            labels[4, :] *= width
-            labels[3, :] *= height
-            labels[5, :] *= height
-            labels[4, :] += labels[2, :]
-            labels[5, :] += labels[3, :]
 
+            labels = np.loadtxt(weak_label_path)
+            # print('weak loaded', labels[:3, 2:])
+            labels[:, 2] = labels[:, 2] * width
+            labels[:, 4] = labels[:, 4] * width
+            labels[:, 3] = labels[:, 3] * height
+            labels[:, 5] = labels[:, 5] * height
+            labels[:, 4] += labels[:, 2]
+            labels[:, 5] += labels[:, 3]
+            # print('weak', labels[:3, 2:])
+
+            res = np.zeros_like(labels)
             res[:, 0:4] = labels[:, -4:]
             res[:, 5] = labels[:, 1]
-            # all are from class one
-            # res[:, 4] = labels[:, 0]
+        # sys.stderr.write('weak shape ' + str(res.shape) + '\n values \n' + str(res))
+
+        # all are from class one
+        # res[:, 4] = labels[:, 0]
 
         img = cv2.imread(img_file)
         if img is None:

yolox/evaluators/mot_evaluator.py

@@ -88,6 +88,7 @@ class MOTEvaluator:
             test_size=None,
             result_folder=None,
             adaptation_period=None,
+            eval_det=True,
     ):
         """
         COCO average precision (AP) Evaluation. Iterate inference on the test dataset
@@ -103,17 +104,22 @@ class MOTEvaluator:
             ap50 (float) : COCO AP of IoU=50
             summary (sr): summary info of evaluation.
         """
-
+        tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+        model = model.type(tensor_type)
         if adaptation_period is not None:
             logger.info('cloning model...')
             learner = model.clone()
         else:
             learner = model
         # TODO half to amp_test
-        tensor_type = torch.cuda.HalfTensor if half else torch.cuda.FloatTensor
+        self.scaler = torch.cuda.amp.GradScaler(enabled=half,init_scale=8192)
+
         learner = learner.eval()
+        self.amp_training = False
         if half:
+            logger.info('half...')
             learner = learner.half()
+            self.amp_training = True
         ids = []
         data_list = []
         results = []
@@ -191,13 +197,21 @@ class MOTEvaluator:
             if is_time_record:
                 start = time.time()
             if adaptation_period is not None and cur_iter % adaptation_period == 0:
-                learner.train()
-                targets = targets.type(tensor_type)
-                targets.requires_grad = False
-                outputs = learner(imgs, targets)
-                loss = outputs["total_loss"]
+                with torch.cuda.amp.autocast(enabled=self.amp_training):
+                    learner.train()
+                    targets = targets.type(tensor_type)
+                    targets.requires_grad = False
+                    outputs = learner(imgs, targets)
+                    loss = outputs["total_loss"]
+                    # loss = outputs["iou_loss"]
+                # logger.info("loss Norm: {} , scale {}".format(torch.norm(loss), self.scaler.get_scale()))
+                loss = self.scaler.scale(loss)
+                # logger.info("loss Norm: {} , scale {}".format(torch.norm(loss), self.scaler.get_scale()))
                 learner.adapt(loss)
+                # self.scaler.update()
+                # learner.adapt(self.scaler.scale(loss))
                 learner.eval()
+                learner.half()
 
             with torch.no_grad():
                 outputs = learner(imgs)
@@ -244,8 +258,10 @@ class MOTEvaluator:
             data_list = gather(data_list, dst=0)
             data_list = list(itertools.chain(*data_list))
             torch.distributed.reduce(statistics, dst=0)
-
-        eval_results = self.evaluate_prediction(data_list, statistics)
+        if eval_det:
+            eval_results = self.evaluate_prediction(data_list, statistics)
+        else:
+            eval_results = 0, 0, 'skipped'
         synchronize()
         return eval_results
 

yolox/exp/base_meta_exp.py

@@ -48,7 +48,7 @@ class BaseMetaExp(metaclass=ABCMeta):
         pass
 
     @abstractmethod
-    def get_evaluator(self):
+    def get_evaluators(self):
         pass
 
     @abstractmethod

yolox/exp/meta_yolox_base.py

@@ -63,6 +63,7 @@ class MetaExp(BaseMetaExp):
         # -----------------  Meta-learning ------------------ #
         self.first_order = True
         self.inner_lr = 1e-6
+        # self.inner_lr = 1e-10
 
     def get_model(self):
         from yolox.models import YOLOPAFPN, YOLOX, YOLOXHead

yolox/layers/fast_coco_eval_api.py

@@ -7,7 +7,7 @@
 
 import numpy as np
 from pycocotools.cocoeval import COCOeval
-
+##TODO: check this
 # import torch first to make yolox._C work without ImportError of libc10.so
 # in YOLOX, env is already set in __init__.py.
 from yolox import _C

yolox/statics.py

@@ -0,0 +1,5 @@
+# Mahdi Abdollahpour, 16/12/2021, 10:22 AM, PyCharm, ByteTrack
+
+
+
+DATA_PATH = '/home/abdollahpour.ce.sharif/ByteTrackData'