Loading
Feedback

AI Blitz #7

Solution for Debris Detection

Solution for Debris Detection

By derinformatiker


In [1]:
!git clone https://github.com/derInformatiker/AIcrowd-AIBlitz7-Solution.git
!pip install -r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt
!pip install aicrowd-cli==0.1
Cloning into 'AIcrowd-AIBlitz7-Solution'...
remote: warning: multi-pack bitmap is missing required reverse index
remote: Enumerating objects: 305, done.
remote: Counting objects: 100% (305/305), done.
remote: Compressing objects: 100% (179/179), done.
remote: Total 305 (delta 177), reused 219 (delta 121), pack-reused 0
Receiving objects: 100% (305/305), 456.81 KiB | 24.04 MiB/s, done.
Resolving deltas: 100% (177/177), done.
Collecting pandas==1.0.5
  Downloading https://files.pythonhosted.org/packages/af/f3/683bf2547a3eaeec15b39cef86f61e921b3b187f250fcd2b5c5fb4386369/pandas-1.0.5-cp37-cp37m-manylinux1_x86_64.whl (10.1MB)
     |████████████████████████████████| 10.1MB 18.8MB/s 
Collecting numpy==1.20.2
  Downloading https://files.pythonhosted.org/packages/73/ef/8967d406f3f85018ceb5efab50431e901683188f1741ceb053efcab26c87/numpy-1.20.2-cp37-cp37m-manylinux2010_x86_64.whl (15.3MB)
     |████████████████████████████████| 15.3MB 207kB/s 
Collecting opencv-python==4.2.0.32
  Downloading https://files.pythonhosted.org/packages/34/a3/403dbaef909fee9f9f6a8eaff51d44085a14e5bb1a1ff7257117d744986a/opencv_python-4.2.0.32-cp37-cp37m-manylinux1_x86_64.whl (28.2MB)
     |████████████████████████████████| 28.2MB 168kB/s 
Collecting tqdm==4.59.0
  Downloading https://files.pythonhosted.org/packages/f8/3e/2730d0effc282960dbff3cf91599ad0d8f3faedc8e75720fdf224b31ab24/tqdm-4.59.0-py2.py3-none-any.whl (74kB)
     |████████████████████████████████| 81kB 10.1MB/s 
Collecting detecto==1.2.1
  Downloading https://files.pythonhosted.org/packages/94/de/2d4bbe9a16092e4643f89e3f2d147893609c74bc4c54f29448d3314a4bfc/detecto-1.2.1-py3-none-any.whl
Collecting matplotlib==3.3.0
  Downloading https://files.pythonhosted.org/packages/1c/15/3fea1bfb7e5b77b7cca9c6010a9cabc58ea125385345ecb6f5832eb8b49a/matplotlib-3.3.0-1-cp37-cp37m-manylinux1_x86_64.whl (11.5MB)
     |████████████████████████████████| 11.5MB 47.4MB/s 
Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas==1.0.5->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 1)) (2018.9)
Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from pandas==1.0.5->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 1)) (2.8.1)
Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from detecto==1.2.1->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 5)) (0.9.1+cu101)
Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (from detecto==1.2.1->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 5)) (1.8.1+cu101)
Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.0->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 6)) (7.1.2)
Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.3 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.0->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 6)) (2.4.7)
Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.0->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 6)) (0.10.0)
Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib==3.3.0->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 6)) (1.3.1)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.6.1->pandas==1.0.5->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 1)) (1.15.0)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch->detecto==1.2.1->-r AIcrowd-AIBlitz7-Solution/challenge2/requirements.txt (line 5)) (3.7.4.3)
ERROR: tensorflow 2.4.1 has requirement numpy~=1.19.2, but you'll have numpy 1.20.2 which is incompatible.
ERROR: google-colab 1.0.0 has requirement pandas~=1.1.0; python_version >= "3.0", but you'll have pandas 1.0.5 which is incompatible.
ERROR: datascience 0.10.6 has requirement folium==0.2.1, but you'll have folium 0.8.3 which is incompatible.
ERROR: albumentations 0.1.12 has requirement imgaug<0.2.7,>=0.2.5, but you'll have imgaug 0.2.9 which is incompatible.
Installing collected packages: numpy, pandas, opencv-python, tqdm, matplotlib, detecto
  Found existing installation: numpy 1.19.5
    Uninstalling numpy-1.19.5:
      Successfully uninstalled numpy-1.19.5
  Found existing installation: pandas 1.1.5
    Uninstalling pandas-1.1.5:
      Successfully uninstalled pandas-1.1.5
  Found existing installation: opencv-python 4.1.2.30
    Uninstalling opencv-python-4.1.2.30:
      Successfully uninstalled opencv-python-4.1.2.30
  Found existing installation: tqdm 4.41.1
    Uninstalling tqdm-4.41.1:
      Successfully uninstalled tqdm-4.41.1
  Found existing installation: matplotlib 3.2.2
    Uninstalling matplotlib-3.2.2:
      Successfully uninstalled matplotlib-3.2.2
Successfully installed detecto-1.2.1 matplotlib-3.3.0 numpy-1.20.2 opencv-python-4.2.0.32 pandas-1.0.5 tqdm-4.59.0
Collecting aicrowd-cli==0.1
  Downloading https://files.pythonhosted.org/packages/84/45/232809ceb8b1462a2c7eec17335cef9e74bdd0fbde6d5d31a9fb570f1bcf/aicrowd-cli-0.1.0.tar.gz (40kB)
     |████████████████████████████████| 40kB 6.3MB/s 
Requirement already satisfied: click<8,>=7.1.2 in /usr/local/lib/python3.7/dist-packages (from aicrowd-cli==0.1) (7.1.2)
Collecting gitpython
  Downloading https://files.pythonhosted.org/packages/a6/99/98019716955ba243657daedd1de8f3a88ca1f5b75057c38e959db22fb87b/GitPython-3.1.14-py3-none-any.whl (159kB)
     |████████████████████████████████| 163kB 24.5MB/s 
Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from aicrowd-cli==0.1) (2.23.0)
Collecting requests-toolbelt
  Downloading https://files.pythonhosted.org/packages/60/ef/7681134338fc097acef8d9b2f8abe0458e4d87559c689a8c306d0957ece5/requests_toolbelt-0.9.1-py2.py3-none-any.whl (54kB)
     |████████████████████████████████| 61kB 10.5MB/s 
Collecting rich
  Downloading https://files.pythonhosted.org/packages/1a/da/2a1f064dc620ab47f3f826ae085384084b71ea05c8c21d67f1dfc29189ab/rich-10.1.0-py3-none-any.whl (201kB)
     |████████████████████████████████| 204kB 53.2MB/s 
Requirement already satisfied: toml in /usr/local/lib/python3.7/dist-packages (from aicrowd-cli==0.1) (0.10.2)
Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from aicrowd-cli==0.1) (4.59.0)
Collecting gitdb<5,>=4.0.1
  Downloading https://files.pythonhosted.org/packages/ea/e8/f414d1a4f0bbc668ed441f74f44c116d9816833a48bf81d22b697090dba8/gitdb-4.0.7-py3-none-any.whl (63kB)
     |████████████████████████████████| 71kB 11.5MB/s 
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->aicrowd-cli==0.1) (2.10)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->aicrowd-cli==0.1) (1.24.3)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->aicrowd-cli==0.1) (2020.12.5)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->aicrowd-cli==0.1) (3.0.4)
Collecting colorama<0.5.0,>=0.4.0
  Downloading https://files.pythonhosted.org/packages/44/98/5b86278fbbf250d239ae0ecb724f8572af1c91f4a11edf4d36a206189440/colorama-0.4.4-py2.py3-none-any.whl
Requirement already satisfied: typing-extensions<4.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from rich->aicrowd-cli==0.1) (3.7.4.3)
Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from rich->aicrowd-cli==0.1) (2.6.1)
Collecting commonmark<0.10.0,>=0.9.0
  Downloading https://files.pythonhosted.org/packages/b1/92/dfd892312d822f36c55366118b95d914e5f16de11044a27cf10a7d71bbbf/commonmark-0.9.1-py2.py3-none-any.whl (51kB)
     |████████████████████████████████| 51kB 9.0MB/s 
Collecting smmap<5,>=3.0.1
  Downloading https://files.pythonhosted.org/packages/68/ee/d540eb5e5996eb81c26ceffac6ee49041d473bc5125f2aa995cf51ec1cf1/smmap-4.0.0-py2.py3-none-any.whl
Building wheels for collected packages: aicrowd-cli
  Building wheel for aicrowd-cli (setup.py) ... done
  Created wheel for aicrowd-cli: filename=aicrowd_cli-0.1.0-cp37-none-any.whl size=34929 sha256=745f5ecd9fadfc7b07c82093b1fd5c23da3d1359a34a2c14d5ce6035f31faf03
  Stored in directory: /root/.cache/pip/wheels/7e/ec/cc/a38391f23ad9bd044c91cd5e6e0a30f2a4265a1eb5adedf9f5
Successfully built aicrowd-cli
Installing collected packages: smmap, gitdb, gitpython, requests-toolbelt, colorama, commonmark, rich, aicrowd-cli
Successfully installed aicrowd-cli-0.1.0 colorama-0.4.4 commonmark-0.9.1 gitdb-4.0.7 gitpython-3.1.14 requests-toolbelt-0.9.1 rich-10.1.0 smmap-4.0.0

RESTART RUNTIME TO USE NEW PACKAGES

In [1]:
API_KEY = ""  # Please enter your API Key from [https://www.aicrowd.com/participants/me]
!aicrowd login --api-key $API_KEY
API Key valid
Saved API Key successfully!
In [2]:
!aicrowd dataset download --challenge debris-detection

!rm -rf data
!mkdir data

!unzip -q train.zip  -d data/train
!unzip -q val.zip -d data/val
!unzip -q test.zip  -d data/test

!mv train.csv data/train.csv
!mv val.csv data/val.csv
!mv sample_submission.csv data/sample_submission.csv
sample_submission.csv: 100% 1.98M/1.98M [00:00<00:00, 4.18MB/s]
test.zip: 100% 259M/259M [00:33<00:00, 7.71MB/s]
train.csv: 100% 1.78M/1.78M [00:00<00:00, 3.67MB/s]
train.zip: 100% 1.04G/1.04G [02:11<00:00, 7.88MB/s]
val.csv: 100% 176k/176k [00:00<00:00, 764kB/s]
val.zip: 100% 104M/104M [00:18<00:00, 5.70MB/s]
In [3]:
import pandas as pd
import ast
import numpy as np
from detecto import core, utils, visualize
from matplotlib import pyplot as plt
from tqdm import tqdm
import cv2
In [4]:
data = pd.read_csv('data/train.csv')
fn, w, h, cl, boxes, imgid = [], [], [], [], [], []
counter = 0
for i in range(300):
    idx = int(data['ImageID'].iloc[i])
    bbox = ast.literal_eval(data['bboxes'].iloc[i])
    for box in bbox:
        fn.append(f'{idx}.jpg')
        w.append(512)
        h.append(512)
        cl.append('debris')
        boxes.append(box)
        imgid.append(counter)
        counter += 1
boxes = np.array(boxes).T
pd.DataFrame(
    {'filename':fn, 
     'width' : w, 
     'height' : h, 
     'class' : cl,
     'xmin' : boxes[0],
     'ymin' : boxes[2],
     'xmax' : boxes[1],
     'ymax':boxes[3],
     'image_id':imgid
    }
).to_csv('data/labels.csv',index = False)
In [6]:
dataset = core.Dataset('data/labels.csv','data/train')
model = core.Model(['debris'],pretrained = False)
m = model.fit(dataset,epochs=10, learning_rate=0.001, lr_step_size=5,verbose = True)
Downloading: "https://download.pytorch.org/models/resnet50-19c8e357.pth" to /root/.cache/torch/hub/checkpoints/resnet50-19c8e357.pth
  0%|          | 0/1137 [00:00<?, ?it/s]
Epoch 1 of 1
Begin iterating over training dataset
100%|██████████| 1137/1137 [03:56<00:00,  4.81it/s]
In [ ]:
out = []
for i in tqdm(range(5000)):
    image = utils.read_image(f'data/test/{i}.jpg')
    labels, boxes, scores = model.predict(image)
    bb = []
    for u,box in enumerate(boxes):
        box = box.cpu().tolist()
        bb.append([box[0],box[2],box[1],box[3],float(scores[u]+0.2 if scores[u]+0.2 <= 1 else 1)])
    out.append(bb)
In [ ]:
def draw_bboxes(img, bboxes, train,color=(255, 0, 0), thickness=1):
    for u, bbox in enumerate(bboxes):
        # if [x1, y1, x2, y2]
        img = cv2.rectangle(img, (int(bbox[0]),int(bbox[2])), (int(bbox[1]),int(bbox[3])), color, thickness)
    return img
In [ ]:
num = 5
i = utils.read_image(f'data/test/{num}.jpg')
#i = cv2.flip(i,0)
image = draw_bboxes(i,out[num],True)
#image = draw_bboxes(image,t[num],False,(0,255,0))
plt.clf()
plt.figure(figsize=(10,10))
plt.imshow(image)
In [ ]:
df = pd.read_csv('data/sample_submission.csv')
df.bboxes = out
df.to_csv('submission.csv',index = False)
In [ ]:

↕️  Read More


Comments

You must login before you can post a comment.