Merge pull request #14627 from l-bat:demo_kinetics

* Support 3D ResNet-34-kinetics * Update sample * Remove preprocess * Change test * Fix sample
6 years ago · f94253b82b
parent 8ea2f59d33
commit f94253b82b
3 changed files with 540 additions and 3 deletions
--- a/modules/dnn/test/test_onnx_importer.cpp
+++ b/modules/dnn/test/test_onnx_importer.cpp
@ -352,7 +352,7 @@ TEST_P(Test_ONNX_nets, ResNet18v1)
    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
    // output range: [-16; 22], after Softmax [0, 0.51]
-    testONNXModels("resnet18v1", pb, default_l1, default_lInf, true);
+    testONNXModels("resnet18v1", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD);
 }
 TEST_P(Test_ONNX_nets, ResNet50v1)
@ -360,7 +360,7 @@ TEST_P(Test_ONNX_nets, ResNet50v1)
    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
    // output range: [-67; 75], after Softmax [0, 0.98]
-    testONNXModels("resnet50v1", pb, default_l1, default_lInf, true);
+    testONNXModels("resnet50v1", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD);
 }
 TEST_P(Test_ONNX_nets, ResNet101_DUC_HDC)
@ -477,7 +477,7 @@ TEST_P(Test_ONNX_nets, DenseNet121)
    applyTestTag(CV_TEST_TAG_MEMORY_512MB);
    // output range: [-87; 138], after Softmax [0; 1]
-    testONNXModels("densenet121", pb, default_l1, default_lInf, true);
+    testONNXModels("densenet121", pb, default_l1, default_lInf, true, target != DNN_TARGET_MYRIAD);
 }
 TEST_P(Test_ONNX_nets, Inception_v1)
@ -497,6 +497,61 @@ TEST_P(Test_ONNX_nets, Shufflenet)
    testONNXModels("shufflenet", pb);
 }
 TEST_P(Test_ONNX_nets, Resnet34_kinetics)
 {
    if (backend != DNN_BACKEND_INFERENCE_ENGINE || target != DNN_TARGET_CPU)
        throw SkipTestException("Only DLIE backend on CPU is supported");
    String onnxmodel = findDataFile("dnn/resnet-34_kinetics.onnx");
    Mat image0 = imread(findDataFile("dnn/dog416.png"));
    Mat image1 = imread(findDataFile("dnn/street.png"));
    Mat ref0 = blobFromNPY(_tf("data/output_kinetics0.npy"));
    Mat ref1 = blobFromNPY(_tf("data/output_kinetics1.npy"));
    std::vector<Mat> images_0(16, image0);
    std::vector<Mat> images_1(16, image1);
    Mat blob0 = blobFromImages(images_0, 1.0, Size(112, 112), Scalar(114.7748, 107.7354, 99.4750), true, true);
    Mat blob1 = blobFromImages(images_1, 1.0, Size(112, 112), Scalar(114.7748, 107.7354, 99.4750), true, true);
    Net permute;
    LayerParams lp;
    int order[] = {1, 0, 2, 3};
    lp.set("order", DictValue::arrayInt<int*>(&order[0], 4));
    permute.addLayerToPrev("perm", "Permute", lp);
    permute.setInput(blob0);
    Mat input0 = permute.forward().clone();
    permute.setInput(blob1);
    Mat input1 = permute.forward().clone();
    int dims[] = {1, 3, 16, 112, 112};
    input0 = input0.reshape(0, 5, &dims[0]);
    input1 = input1.reshape(0, 5, &dims[0]);
    Net net = readNetFromONNX(onnxmodel);
    ASSERT_FALSE(net.empty());
    net.setPreferableBackend(backend);
    net.setPreferableTarget(target);
    // output range [-5, 11]
    float l1 = 0.0013;
    float lInf = 0.009;
    checkBackend(&input0, &ref0);
    net.setInput(input0);
    Mat out = net.forward().clone();
    normAssert(ref0, out, "", l1, lInf);
    checkBackend(&input1, &ref1);
    net.setInput(input1);
    out = net.forward().clone();
    normAssert(ref1, out, "", l1, lInf);
    expectNoFallbacksFromIE(net);
 }
 INSTANTIATE_TEST_CASE_P(/**/, Test_ONNX_nets, dnnBackendsAndTargets());
 }} // namespace
--- a/samples/data/dnn/action_recongnition_kinetics.txt
+++ b/samples/data/dnn/action_recongnition_kinetics.txt
@ -0,0 +1,400 @@
 abseiling
 air drumming
 answering questions
 applauding
 applying cream
 archery
 arm wrestling
 arranging flowers
 assembling computer
 auctioning
 baby waking up
 baking cookies
 balloon blowing
 bandaging
 barbequing
 bartending
 beatboxing
 bee keeping
 belly dancing
 bench pressing
 bending back
 bending metal
 biking through snow
 blasting sand
 blowing glass
 blowing leaves
 blowing nose
 blowing out candles
 bobsledding
 bookbinding
 bouncing on trampoline
 bowling
 braiding hair
 breading or breadcrumbing
 breakdancing
 brush painting
 brushing hair
 brushing teeth
 building cabinet
 building shed
 bungee jumping
 busking
 canoeing or kayaking
 capoeira
 carrying baby
 cartwheeling
 carving pumpkin
 catching fish
 catching or throwing baseball
 catching or throwing frisbee
 catching or throwing softball
 celebrating
 changing oil
 changing wheel
 checking tires
 cheerleading
 chopping wood
 clapping
 clay pottery making
 clean and jerk
 cleaning floor
 cleaning gutters
 cleaning pool
 cleaning shoes
 cleaning toilet
 cleaning windows
 climbing a rope
 climbing ladder
 climbing tree
 contact juggling
 cooking chicken
 cooking egg
 cooking on campfire
 cooking sausages
 counting money
 country line dancing
 cracking neck
 crawling baby
 crossing river
 crying
 curling hair
 cutting nails
 cutting pineapple
 cutting watermelon
 dancing ballet
 dancing charleston
 dancing gangnam style
 dancing macarena
 deadlifting
 decorating the christmas tree
 digging
 dining
 disc golfing
 diving cliff
 dodgeball
 doing aerobics
 doing laundry
 doing nails
 drawing
 dribbling basketball
 drinking
 drinking beer
 drinking shots
 driving car
 driving tractor
 drop kicking
 drumming fingers
 dunking basketball
 dying hair
 eating burger
 eating cake
 eating carrots
 eating chips
 eating doughnuts
 eating hotdog
 eating ice cream
 eating spaghetti
 eating watermelon
 egg hunting
 exercising arm
 exercising with an exercise ball
 extinguishing fire
 faceplanting
 feeding birds
 feeding fish
 feeding goats
 filling eyebrows
 finger snapping
 fixing hair
 flipping pancake
 flying kite
 folding clothes
 folding napkins
 folding paper
 front raises
 frying vegetables
 garbage collecting
 gargling
 getting a haircut
 getting a tattoo
 giving or receiving award
 golf chipping
 golf driving
 golf putting
 grinding meat
 grooming dog
 grooming horse
 gymnastics tumbling
 hammer throw
 headbanging
 headbutting
 high jump
 high kick
 hitting baseball
 hockey stop
 holding snake
 hopscotch
 hoverboarding
 hugging
 hula hooping
 hurdling
 hurling (sport)
 ice climbing
 ice fishing
 ice skating
 ironing
 javelin throw
 jetskiing
 jogging
 juggling balls
 juggling fire
 juggling soccer ball
 jumping into pool
 jumpstyle dancing
 kicking field goal
 kicking soccer ball
 kissing
 kitesurfing
 knitting
 krumping
 laughing
 laying bricks
 long jump
 lunge
 making a cake
 making a sandwich
 making bed
 making jewelry
 making pizza
 making snowman
 making sushi
 making tea
 marching
 massaging back
 massaging feet
 massaging legs
 massaging person's head
 milking cow
 mopping floor
 motorcycling
 moving furniture
 mowing lawn
 news anchoring
 opening bottle
 opening present
 paragliding
 parasailing
 parkour
 passing American football (in game)
 passing American football (not in game)
 peeling apples
 peeling potatoes
 petting animal (not cat)
 petting cat
 picking fruit
 planting trees
 plastering
 playing accordion
 playing badminton
 playing bagpipes
 playing basketball
 playing bass guitar
 playing cards
 playing cello
 playing chess
 playing clarinet
 playing controller
 playing cricket
 playing cymbals
 playing didgeridoo
 playing drums
 playing flute
 playing guitar
 playing harmonica
 playing harp
 playing ice hockey
 playing keyboard
 playing kickball
 playing monopoly
 playing organ
 playing paintball
 playing piano
 playing poker
 playing recorder
 playing saxophone
 playing squash or racquetball
 playing tennis
 playing trombone
 playing trumpet
 playing ukulele
 playing violin
 playing volleyball
 playing xylophone
 pole vault
 presenting weather forecast
 pull ups
 pumping fist
 pumping gas
 punching bag
 punching person (boxing)
 push up
 pushing car
 pushing cart
 pushing wheelchair
 reading book
 reading newspaper
 recording music
 riding a bike
 riding camel
 riding elephant
 riding mechanical bull
 riding mountain bike
 riding mule
 riding or walking with horse
 riding scooter
 riding unicycle
 ripping paper
 robot dancing
 rock climbing
 rock scissors paper
 roller skating
 running on treadmill
 sailing
 salsa dancing
 sanding floor
 scrambling eggs
 scuba diving
 setting table
 shaking hands
 shaking head
 sharpening knives
 sharpening pencil
 shaving head
 shaving legs
 shearing sheep
 shining shoes
 shooting basketball
 shooting goal (soccer)
 shot put
 shoveling snow
 shredding paper
 shuffling cards
 side kick
 sign language interpreting
 singing
 situp
 skateboarding
 ski jumping
 skiing (not slalom or crosscountry)
 skiing crosscountry
 skiing slalom
 skipping rope
 skydiving
 slacklining
 slapping
 sled dog racing
 smoking
 smoking hookah
 snatch weight lifting
 sneezing
 sniffing
 snorkeling
 snowboarding
 snowkiting
 snowmobiling
 somersaulting
 spinning poi
 spray painting
 spraying
 springboard diving
 squat
 sticking tongue out
 stomping grapes
 stretching arm
 stretching leg
 strumming guitar
 surfing crowd
 surfing water
 sweeping floor
 swimming backstroke
 swimming breast stroke
 swimming butterfly stroke
 swing dancing
 swinging legs
 swinging on something
 sword fighting
 tai chi
 taking a shower
 tango dancing
 tap dancing
 tapping guitar
 tapping pen
 tasting beer
 tasting food
 testifying
 texting
 throwing axe
 throwing ball
 throwing discus
 tickling
 tobogganing
 tossing coin
 tossing salad
 training dog
 trapezing
 trimming or shaving beard
 trimming trees
 triple jump
 tying bow tie
 tying knot (not on a tie)
 tying tie
 unboxing
 unloading truck
 using computer
 using remote controller (not gaming)
 using segway
 vault
 waiting in line
 walking the dog
 washing dishes
 washing feet
 washing hair
 washing hands
 water skiing
 water sliding
 watering plants
 waxing back
 waxing chest
 waxing eyebrows
 waxing legs
 weaving basket
 welding
 whistling
 windsurfing
 wrapping present
 wrestling
 writing
 yawning
 yoga
 zumba
--- a/samples/dnn/action_recognition.py
+++ b/samples/dnn/action_recognition.py
@ -0,0 +1,82 @@
 import os
 import numpy as np
 import cv2 as cv
 import argparse
 from common import findFile
 parser = argparse.ArgumentParser(description='Use this script to run action recognition using 3D ResNet34',
                                 formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument('--input', '-i', help='Path to input video file. Skip this argument to capture frames from a camera.')
 parser.add_argument('--model', required=True, help='Path to model.')
 parser.add_argument('--classes', default=findFile('action_recongnition_kinetics.txt'), help='Path to classes list.')
 # To get net download original repository https://github.com/kenshohara/video-classification-3d-cnn-pytorch
 # For correct ONNX export modify file: video-classification-3d-cnn-pytorch/models/resnet.py
 # change
 # - def downsample_basic_block(x, planes, stride):
 # -     out = F.avg_pool3d(x, kernel_size=1, stride=stride)
 # -     zero_pads = torch.Tensor(out.size(0), planes - out.size(1),
 # -                              out.size(2), out.size(3),
 # -                              out.size(4)).zero_()
 # -     if isinstance(out.data, torch.cuda.FloatTensor):
 # -         zero_pads = zero_pads.cuda()
 # -
 # -     out = Variable(torch.cat([out.data, zero_pads], dim=1))
 # -     return out
 # To
 # + def downsample_basic_block(x, planes, stride):
 # +     out = F.avg_pool3d(x, kernel_size=1, stride=stride)
 # +     out = F.pad(out, (0, 0, 0, 0, 0, 0, 0, int(planes - out.size(1)), 0, 0), "constant", 0)
 # +     return out
 # To ONNX export use torch.onnx.export(model, inputs, model_name)
 def get_class_names(path):
    class_names = []
    with open(path) as f:
        for row in f:
            class_names.append(row[:-1])
    return class_names
 def classify_video(video_path, net_path):
    SAMPLE_DURATION = 16
    SAMPLE_SIZE = 112
    mean = (114.7748, 107.7354, 99.4750)
    class_names = get_class_names(args.classes)
    net = cv.dnn.readNet(net_path)
    net.setPreferableBackend(cv.dnn.DNN_BACKEND_INFERENCE_ENGINE)
    net.setPreferableTarget(cv.dnn.DNN_TARGET_CPU)
    winName = 'Deep learning image classification in OpenCV'
    cv.namedWindow(winName, cv.WINDOW_AUTOSIZE)
    cap = cv.VideoCapture(video_path)
    while cv.waitKey(1) < 0:
        frames = []
        for _ in range(SAMPLE_DURATION):
            hasFrame, frame = cap.read()
            if not hasFrame:
                exit(0)
            frames.append(frame)
        inputs = cv.dnn.blobFromImages(frames, 1, (SAMPLE_SIZE, SAMPLE_SIZE), mean, True, crop=True)
        inputs = np.transpose(inputs, (1, 0, 2, 3))
        inputs = np.expand_dims(inputs, axis=0)
        net.setInput(inputs)
        outputs = net.forward()
        class_pred = np.argmax(outputs)
        label = class_names[class_pred]
        for frame in frames:
            labelSize, baseLine = cv.getTextSize(label, cv.FONT_HERSHEY_SIMPLEX, 0.5, 1)
            cv.rectangle(frame, (0, 10 - labelSize[1]),
                                (labelSize[0], 10 + baseLine), (255, 255, 255), cv.FILLED)
            cv.putText(frame, label, (0, 10), cv.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0))
            cv.imshow(winName, frame)
        if cv.waitKey(1) & 0xFF == ord('q'):
            break
 if __name__ == "__main__":
    args, _ = parser.parse_known_args()
    classify_video(args.input if args.input else 0, args.model)