--- a
+++ b/LSTM - Experiments/our-estimation.ipynb
@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyNYFs7ypR4vSbzCNaLRYFsy"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","source":["# !git clone https://github.com/SkalskiP/yolov7.git\n","# %cd yolov7\n","# !git checkout fix/problems_associated_with_the_latest_versions_of_pytorch_and_numpy\n","# !pip install -r requirements.txt"],"metadata":{"id":"qpTM9uLPScRW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["YOLO_DIR = 'yolov7'"],"metadata":{"id":"IKogxJ5KGIUW","executionInfo":{"status":"ok","timestamp":1692369808437,"user_tz":300,"elapsed":311,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":13,"outputs":[]},{"cell_type":"code","source":["import sys\n","sys.path.insert(0, \"..\")"],"metadata":{"id":"O2wr3an-YwcO","executionInfo":{"status":"ok","timestamp":1692369390529,"user_tz":300,"elapsed":1,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["if not os.path.exists(YOLO_DIR):\n","    print(\"Starting to clone the YoloV7 repo\\n\")\n","    !git clone https://github.com/WongKinYiu/yolov7.git\n","    print(\"\\nDone\")\n","\n","if os.path.exists(YOLO_DIR):\n","    print(\"\\nDirectory is present:\\n\")\n","    !ls yolov7"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"C63T2oKYEwK5","executionInfo":{"status":"ok","timestamp":1692369407005,"user_tz":300,"elapsed":4768,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"bbd57dfa-ce18-4185-a644-3fbedf0040a8"},"execution_count":5,"outputs":[{"output_type":"stream","name":"stdout","text":["Starting to clone the YoloV7 repo\n","\n","Cloning into 'yolov7'...\n","remote: Enumerating objects: 1191, done.\u001b[K\n","remote: Counting objects: 100% (6/6), done.\u001b[K\n","remote: Compressing objects: 100% (4/4), done.\u001b[K\n","remote: Total 1191 (delta 2), reused 6 (delta 2), pack-reused 1185\u001b[K\n","Receiving objects: 100% (1191/1191), 74.23 MiB | 25.91 MiB/s, done.\n","Resolving deltas: 100% (511/511), done.\n","\n","Done\n","\n","Directory is present:\n","\n","cfg\tdetect.py  hubconf.py  models\t  requirements.txt  tools\t  utils\n","data\texport.py  inference   paper\t  scripts\t    train_aux.py\n","deploy\tfigure\t   LICENSE.md  README.md  test.py\t    train.py\n"]}]},{"cell_type":"code","source":["RESULTS_DIR = 'results'\n","\n","if not os.path.exists(RESULTS_DIR):\n","    print(\"Creating the directory:\", RESULTS_DIR)\n","    os.mkdir(RESULTS_DIR)\n","    print(\"Done\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4IEbGE_lFMZ4","executionInfo":{"status":"ok","timestamp":1692369482445,"user_tz":300,"elapsed":356,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"5f7bb01d-a2af-4fc2-a88e-a24be5b8930f"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stdout","text":["Creating the directory: results\n","Done\n"]}]},{"cell_type":"code","source":["YOLOV7_MODEL = [\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-tiny.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7x.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-w6.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-e6.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-d6.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-e6e.pt\",\n","    \"https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-w6-pose.pt\",\n","]"],"metadata":{"id":"Fdi-uMQ9F9bE","executionInfo":{"status":"ok","timestamp":1692369753764,"user_tz":300,"elapsed":311,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":10,"outputs":[]},{"cell_type":"code","source":["def get_yolov7_model(modelistid=1):\n","    \"\"\"\n","    Download YoloV7 model from a yoloV7 model list\n","    \"\"\"\n","    modelid = YOLOV7_MODEL[modelistid]\n","\n","    if not os.path.exists(modelid):\n","        print(\"Downloading the model:\",\n","              os.path.basename(modelid), \"from:\", modelid)\n","        urllib.request.urlretrieve(modelid,\n","                                   filename=os.path.basename(modelid))\n","        print(\"Done\\n\")\n","        !ls yolo*.pt -lh\n","\n","    if os.path.exists(modelid):\n","        print(\"Downloaded model files:\")\n","        !ls yolo*.pt -lh"],"metadata":{"id":"yLmjSRcDGUia","executionInfo":{"status":"ok","timestamp":1692369769627,"user_tz":300,"elapsed":358,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":11,"outputs":[]},{"cell_type":"code","execution_count":29,"metadata":{"id":"qJQHtODJ0QOD","colab":{"base_uri":"https://localhost:8080/","height":383},"executionInfo":{"status":"error","timestamp":1692370712622,"user_tz":300,"elapsed":4,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"3bb59d21-9f8b-48ee-ca6a-b3b8a680f664"},"outputs":[{"output_type":"error","ename":"ImportError","evalue":"ignored","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)","\u001b[0;32m<ipython-input-29-3ebd19130d64>\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgeneral\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnon_max_suppression_kpt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mstrip_optimizer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mxyxy2xywh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplots\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0moutput_to_keypoint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mplot_skeleton_kpts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mplot_one_box_kpt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mlogging\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mImportError\u001b[0m: cannot import name 'plot_one_box_kpt' from 'utils.plots' (/content/yolov7/utils/plots.py)","","\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n"],"errorDetails":{"actions":[{"action":"open_url","actionText":"Open Examples","url":"/notebooks/snippets/importing_libraries.ipynb"}]}}],"source":["import torch\n","import os\n","from models.experimental import attempt_load\n","import cv2\n","from utils.datasets import letterbox\n","from torchvision import transforms\n","import numpy as np\n","import time\n","from utils.general import non_max_suppression_kpt,strip_optimizer,xyxy2xywh\n","from utils.plots import output_to_keypoint, plot_skeleton_kpts, colors, plot_one_box_kpt\n","import logging\n","import pandas as pd\n","import itertools"]},{"cell_type":"code","source":["!wget https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-w6-pose.pt"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"erLpYJ19X8Ef","executionInfo":{"status":"ok","timestamp":1692369962314,"user_tz":300,"elapsed":11010,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"934342bf-bc95-4085-ca2c-c234dfdbc751"},"execution_count":19,"outputs":[{"output_type":"stream","name":"stdout","text":["--2023-08-18 14:45:51--  https://github.com/WongKinYiu/yolov7/releases/download/v0.1/yolov7-w6-pose.pt\n","Resolving github.com (github.com)... 140.82.114.3\n","Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n","HTTP request sent, awaiting response... 302 Found\n","Location: https://objects.githubusercontent.com/github-production-release-asset-2e65be/511187726/ad063dcb-fb9a-4511-b4d7-499601326cd8?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230818%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230818T144552Z&X-Amz-Expires=300&X-Amz-Signature=1c8252f73ef2c1330595fd83605f2a11677622662b5be690717c2f38dd217e23&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=511187726&response-content-disposition=attachment%3B%20filename%3Dyolov7-w6-pose.pt&response-content-type=application%2Foctet-stream [following]\n","--2023-08-18 14:45:52--  https://objects.githubusercontent.com/github-production-release-asset-2e65be/511187726/ad063dcb-fb9a-4511-b4d7-499601326cd8?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20230818%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230818T144552Z&X-Amz-Expires=300&X-Amz-Signature=1c8252f73ef2c1330595fd83605f2a11677622662b5be690717c2f38dd217e23&X-Amz-SignedHeaders=host&actor_id=0&key_id=0&repo_id=511187726&response-content-disposition=attachment%3B%20filename%3Dyolov7-w6-pose.pt&response-content-type=application%2Foctet-stream\n","Resolving objects.githubusercontent.com (objects.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n","Connecting to objects.githubusercontent.com (objects.githubusercontent.com)|185.199.108.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 161114789 (154M) [application/octet-stream]\n","Saving to: ‘yolov7-w6-pose.pt’\n","\n","yolov7-w6-pose.pt   100%[===================>] 153.65M  30.0MB/s    in 9.6s    \n","\n","2023-08-18 14:46:02 (16.0 MB/s) - ‘yolov7-w6-pose.pt’ saved [161114789/161114789]\n","\n"]}]},{"cell_type":"code","source":["logging.getLogger().setLevel(logging.ERROR)"],"metadata":{"id":"iKeKV6vQ0eUl","executionInfo":{"status":"ok","timestamp":1692369962314,"user_tz":300,"elapsed":2,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":20,"outputs":[]},{"cell_type":"code","source":["device ='cuda' if torch.cuda.is_available() else 'cpu'\n","model_path = 'yolov7-w6-pose.pt'"],"metadata":{"id":"eOyp5FFN0gkL","executionInfo":{"status":"ok","timestamp":1692369963929,"user_tz":300,"elapsed":1,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":21,"outputs":[]},{"cell_type":"code","source":["def strip_model(device, model_path):\n","    x = torch.load(model_path, map_location=torch.device(device))\n","\n","    if x.get('ema'):\n","        x['model'] = x['ema']  # replace model with ema\n","    for k in 'optimizer', 'training_results', 'wandb_id', 'ema', 'updates':  # keys\n","        x[k] = None\n","    x['epoch'] = -1\n","    if device!='cpu':\n","        x['model'].half()  # to FP16\n","    else:\n","        x['model'].float()\n","    for p in x['model'].parameters():\n","        p.requires_grad = False\n","\n","    torch.save(x, model_path)\n","    mb = os.path.getsize(model_path) / 1E6  # filesize\n","    print(f\"Optimizer stripped from {model_path},{(f' saved as {model_path},') if model_path else ''} {mb:.1f}MB\")"],"metadata":{"id":"26LJXVxw0jP7","executionInfo":{"status":"ok","timestamp":1692369970856,"user_tz":300,"elapsed":301,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":22,"outputs":[]},{"cell_type":"code","source":["strip_model(device, model_path)\n","model = attempt_load(model_path, map_location=device)"],"metadata":{"id":"KgWwR0ZH0k2B","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1692369979571,"user_tz":300,"elapsed":5022,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"e21dcc87-be74-44ae-e381-569ee40947c7"},"execution_count":23,"outputs":[{"output_type":"stream","name":"stdout","text":["Optimizer stripped from yolov7-w6-pose.pt, saved as yolov7-w6-pose.pt, 321.9MB\n","Fusing layers... \n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/torch/functional.py:504: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ../aten/src/ATen/native/TensorShape.cpp:3483.)\n","  return _VF.meshgrid(tensors, **kwargs)  # type: ignore[attr-defined]\n"]}]},{"cell_type":"code","source":["_ = model.eval()\n","names = model.module.names if hasattr(model, 'module') else model.names  # get class names\n","print(f\"Things to identify: {names}\")"],"metadata":{"id":"XcWoZ1EX0mdT","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1692369982444,"user_tz":300,"elapsed":410,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}},"outputId":"a81fca7c-c4b8-420a-d523-19cc6994309a"},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["Things to identify: ['person']\n"]}]},{"cell_type":"code","source":["def load_source(source):\n","    if source.isnumeric() :\n","        cap = cv2.VideoCapture(int(source))    #pass video to videocapture object\n","    else :\n","        cap = cv2.VideoCapture(source)    #pass video to videocapture object\n","    if cap.isOpened() == False:   #check if videocapture not opened\n","        print('Source not found. Check path')\n","    else:\n","        frame_width = int(cap.get(3))  #get video frame width\n","        frame_height = int(cap.get(4)) #get video frame height\n","    return cap"],"metadata":{"id":"vJ-CaP7-0nzo","executionInfo":{"status":"ok","timestamp":1692369982769,"user_tz":300,"elapsed":1,"user":{"displayName":"Francesco Bassino","userId":"12214195385968794219"}}},"execution_count":25,"outputs":[]},{"cell_type":"code","source":["\"\"\"\n","Sequence row:\n","    Video Timestamp Set of landmarks\n","\"\"\"\n","\n","\n","def landmarks_sequence_for_video(video_path, sequence_length=10, separation=6):\n","    \"\"\"\n","        Args\n","        Returns\n","    \"\"\"\n","    sequence_length = sequence_length\n","    separation = separation\n","\n","    count = 0\n","\n","    sequences = [[]]\n","\n","    cap = load_source(video_path)\n","    frame_width = int(cap.get(3))\n","    frame_height = int(cap.get(4))\n","\n","    base_landmarks = None\n","\n","    start = time.time()\n","    current_frame = 0\n","    video_name = video_path.split('/')[-1]\n","    current_group = 1\n","    while(cap.isOpened):\n","        ret, frame = cap.read()\n","\n","        if not ret:\n","            break\n","\n","        count += 1\n","\n","        orig_image = frame #store frame\n","        image = cv2.cvtColor(orig_image, cv2.COLOR_BGR2RGB) #convert frame to RGB\n","        image = letterbox(image, (frame_width), stride=64, auto=True)[0]\n","        image_ = image.copy()\n","        image = transforms.ToTensor()(image)\n","        image = torch.tensor(np.array([image.numpy()]))\n","\n","        image = image.to(device)  #convert image data to device\n","        image = image.float() #convert image to float precision (cpu)\n","\n","\n","        image = image.cpu().squeeze().numpy().transpose((1, 2, 0))\n","\n","        # this frame size works with yolov7, since we don't want to touch their model, we just resize the frame.\n","        desired_width = 640\n","        desired_height = 512\n","        image = cv2.resize(image, (desired_width, desired_height), interpolation=cv2.INTER_LINEAR)\n","        image = image[:desired_height, :desired_width]\n","\n","        # Convert the cropped image back to a torch.Tensor\n","        image = torch.from_numpy(image.transpose((2, 0, 1))).unsqueeze(0).cuda()\n","\n","        with torch.no_grad():  #get predictions\n","            output_data, _ = model(image)\n","\n","        output_data = non_max_suppression_kpt(output_data,   #Apply non max suppression\n","                                        0.25, # Conf. Threshold.\n","                                        0.65, # IoU Threshold.\n","                                        nc=model.yaml['nc'], # Number of classes.\n","                                        nkpt=model.yaml['nkpt'], # Number of keypoints.\n","                                        kpt_label=True)\n","\n","        output = output_to_keypoint(output_data)\n","\n","        im0 = image[0].permute(1, 2, 0) * 255 # Change format [b, c, h, w] to [h, w, c] for displaying the image.\n","        im0 = im0.cpu().numpy().astype(np.uint8)\n","\n","        im0 = cv2.cvtColor(im0, cv2.COLOR_RGB2BGR) #reshape image format to (BGR)\n","        gn = torch.tensor(im0.shape)[[1, 0, 1, 0]]  # normalization gain whwh\n","\n","        for i, pose in enumerate(output_data):  # detections per image\n","            if len(output_data):  #check if no pose\n","                for c in pose[:, 5].unique(): # Print results\n","                    n = (pose[:, 5] == c).sum()  # detections per class\n","\n","                for det_index, (*xyxy, conf, cls) in enumerate(reversed(pose[:,:6])): #loop over poses for drawing on frame\n","                    c = int(cls)  # integer class\n","                    kpts = pose[det_index, 6:]\n","                    label = None # if opt.hide_labels else (names[c] if opt.hide_conf else f'{names[c]} {conf:.2f}')\n","                    plot_one_box_kpt(xyxy, im0, label=label, color=colors(c, True),\n","                                    line_thickness=3,kpt_label=True, kpts=kpts, steps=3,\n","                                    orig_shape=im0.shape[:2])\n","\n","\n","        if count == separation:\n","            # cv2.imshow(\"YOLOv7 Pose Estimation Demo\", im0)\n","\n","            if len(sequences[-1]) >= sequence_length:\n","                sequences += [[]] # init new empty sequence\n","                current_group += 1\n","            else:\n","                # TODO: make sure that the landmarks stored are the desired ones\n","                # Use position difference.\n","                if len(output):\n","                    sequences[-1] += [[video_name, current_group, current_frame, output[0]]]\n","\n","            count = 0\n","\n","\n","        # Press Q on keyboard to  exit\n","        if cv2.waitKey(25) & 0xFF == ord('q'):\n","            break\n","        current_frame += 1\n","\n","    cap.release()\n","    print(f\"\\tfinished after {round(time.time() - start)}s\")\n","    return sequences"],"metadata":{"id":"l62vIwJU0oVi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# landmark retrieval\n","videos = ['../dataset_videos/' + v for v in os.listdir('../dataset_videos') if v.endswith('.mp4')]\n","dataset = []\n","print(videos)"],"metadata":{"id":"vklJI0Hy0r0Y"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i, v in enumerate(videos):\n","    print(f\"Processing video #{i} at '{v}'\")\n","    sequences = landmarks_sequence_for_video(v)\n","    dataset += [sequences]"],"metadata":{"id":"qbnIsGDu0wYc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# this merges every set of landmarks.\n","flatten_data = list(itertools.chain.from_iterable(dataset))\n","flatten_data = list(itertools.chain.from_iterable(flatten_data))"],"metadata":{"id":"_epNpTKW0xGr"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df = pd.DataFrame(columns='video group frame landmarks'.split(), data=flatten_data)"],"metadata":{"id":"IgbF1MnA0ya6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.head()"],"metadata":{"id":"oXIxNxTZ00f3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df.to_csv('dataset.csv', index=False)"],"metadata":{"id":"wk3nctdM01GL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df[df['video'] == 'video3.mp4']"],"metadata":{"id":"tIdH3D-P03V4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"MXNDCRJF04en"},"execution_count":null,"outputs":[]}]}
\ No newline at end of file