Number Plate Detection Using Machine Learning-OpenCV

How many e-challans have you got recently?? The emerging technologies are making this so easy with the help of OpenCV and Machine Learning. Let’s explore how OpenCV and Machine Learning helps in this…

OpenCV:

OpenCV is a huge open-source library for computer vision, machine learning, and image processing. OpenCV supports a wide variety of programming languages like Python, C++, Java, etc. It can process images and videos to identify objects, faces, or even the handwriting of a human. When it is integrated with various libraries, such as Numpy which is a highly optimized library for numerical operations, then the number of weapons increases in your Arsenal i.e whatever operations one can do in Numpy can be combined with OpenCV.

Let’s directly look into the code and understand:

pip3 install opencv-python
pip3 install opencv-contrib-python
pip3 install imutils

• Imutils are a series of convenience functions to make basic image processing functions such as translation, rotation, resizing, skeletonization, and displaying Matplotlib images easier with OpenCV and both Python 2.7 and Python 3

For backend, we will be using Python. we will create a webpage that can

1. Detect the car in a live stream video.
2. Detect the number plate and the Registration number.
3. Fetch the information of the car from the RTO API and display it on the web portal.

Here is the Number plate detector python code :

import cv2
import numpy as np
from imutils.object_detection import non_max_suppression # for image processing and bounding box around the object
#Loading pretrained model
model = cv2.dnn.readNet('frozen_east_text_detection.pb')
model1 = cv2.dnn.readNet('crnn.onnx')
def most_likely(scores, char_set):
    text = ""
    for i in range(scores.shape[0]):
        c = np.argmax(scores[i][0])
        text += char_set[c]
    return text
def map_rule(text):
    char_list = []
    for i in range(len(text)):
        if i == 0:
            if text[i] != '-':
                char_list.append(text[i])
        else:
            if text[i] != '-' and (not (text[i] == text[i - 1])):
                char_list.append(text[i])
    return ''.join(char_list)
def best_path(scores, char_set):
    text = most_likely(scores, char_set)
    final_text = map_rule(text)
    return final_text
alphabet_set = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
blank = '-'
char_set = blank + alphabet_set
def mainfn():
    img = cv2.imread("pic.jpg")
    height, width, _ = img.shape
    new_height = (height//32)*32
    new_width = (width//32)*32
    # get the ratio change in width and height
    h_ratio = height/new_height
    w_ratio = width/new_width
    blob = cv2.dnn.blobFromImage(img,1.0, (new_width, new_height), (123.68, 116.78, 103.94), True, False)
    model.setInput(blob)
    (geometry, scores) = model.forward(model.getUnconnectedOutLayersNames())
    rectangles = []
    confidence_score = []
    for i in range(geometry.shape[2]):
            for j in range(0, geometry.shape[3]):
                if scores[0][0][i][j] < 0.1:
                    continue
                bottom_x = int(j*4 + geometry[0][1][i][j])
                bottom_y = int(i*4 + geometry[0][2][i][j])

                top_x = int(j*4 - geometry[0][3][i][j])
                top_y = int(i*4 - geometry[0][0][i][j])
                rectangles.append((top_x, top_y, bottom_x, bottom_y))
                confidence_score.append(float(scores[0][0][i][j]))
    # use Non-max suppression to get the required rectangles
    fin_boxes = non_max_suppression(np.array(rectangles), probs=confidence_score, overlapThresh=0.5)
    img_copy = img.copy()
    chara = {}
    for (x1, y1, x2, y2) in fin_boxes:
        x1 = int(x1 * w_ratio)
        y1 = int(y1 * h_ratio)
        x2 = int(x2 * w_ratio)
        y2 = int(y2 * h_ratio)
       
        segment = img[y1:y2, x1:x2, :]
        segment_gray = cv2.cvtColor(segment, cv2.COLOR_BGR2GRAY)
        
        blob = cv2.dnn.blobFromImage(segment_gray, scalefactor=1/127.5, size=(100,32), mean=127.5)
        model1.setInput(blob)
        scores = model1.forward()
        text = best_path(scores, char_set)
        chara[x1] = text
        #print(text)
        
        cv2.rectangle(img_copy, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.putText(img_copy, text.strip(), (x1,y1-2), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0,0,255),2)
    final=[]
    for j in sorted(chara):
        final.append(chara[j])
    #print(final)
    s = ""
    s = s.join(final)
    return s
# recognizing text in image(OCR-Optical Character Recognition)
cap = cv2.VideoCapture(0)
count=0
while cap.isOpened():
    
    count=count+1
    ret, img = cap.read()
    if count == 10:
        cv2.imwrite("pic.jpg", img)
        #print("pic clicked")
        output_val = mainfn()
        print(output_val) 
        count=0
    # use multiple of 32 to set the new img shape
    height, width, _ = img.shape
    new_height = (height//32)*32
    new_width = (width//32)*32
    # get the ratio change in width and height
    h_ratio = height/new_height
    w_ratio = width/new_width
    blob = cv2.dnn.blobFromImage(img,1.0, (new_width, new_height), (123.68, 116.78, 103.94), True, False)
    model.setInput(blob)
    (geometry, scores) = model.forward(model.getUnconnectedOutLayersNames())
    rectangles = []
    confidence_score = []
    for i in range(geometry.shape[2]):
        for j in range(0, geometry.shape[3]):
            if scores[0][0][i][j] < 0.1:
                continue
            bottom_x = int(j*4 + geometry[0][1][i][j])
            bottom_y = int(i*4 + geometry[0][2][i][j])

            top_x = int(j*4 - geometry[0][3][i][j])
            top_y = int(i*4 - geometry[0][0][i][j])
            rectangles.append((top_x, top_y, bottom_x, bottom_y))
            confidence_score.append(float(scores[0][0][i][j]))
    # use Non-max suppression to get the required rectangles
    fin_boxes = non_max_suppression(np.array(rectangles), probs=confidence_score, overlapThresh=0.5)
    img_copy = img.copy()
    for (x1, y1, x2, y2) in fin_boxes:
        x1 = int(x1 * w_ratio)
        y1 = int(y1 * h_ratio)
        x2 = int(x2 * w_ratio)
        y2 = int(y2 * h_ratio)
    
        #segment = img[y1:y2+4, x1:x2+2, :]
        segment = img[y1:y2, x1:x2, :]
        segment_gray = cv2.cvtColor(segment, cv2.COLOR_BGR2GRAY)
        
        blob = cv2.dnn.blobFromImage(segment_gray, scalefactor=1/127.5, size=(100,32), mean=127.5)
        model1.setInput(blob)
        scores = model1.forward()
        text = best_path(scores, char_set)
        #print(text)
        cv2.rectangle(img_copy, (x1, y1), (x2, y2), (0, 255, 0), 2)
        cv2.putText(img_copy, text.strip(), (x1,y1-2), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0,0,255),2)
    cv2.imshow("Text Detection", img_copy)
    if cv2.waitKey(1) == 113:
        break
cap.release()
cv2.destroyAllWindows()

Now let’s run this code:

• This code opens the webcam and detects the objects, then OCR tries to read the text from the digital image. Here is the scenario of this:

This reads the text from the image and then prints it on the screen. We then use this number to get the information of the vehicle from the RTO API.

Here is the backend code:

#!/usr/bin/python3
print("content-type: text/html")
print()
import cgi
import requests
import json
BASE = 'http://127.0.0.1:5000/'
field = cgi.FieldStorage()
vehicle_number = field.getvalue("vehicle_number")
response = requests.get(BASE + "vehicle/" + vehicle_number)
responseJson = json.dumps(response.json(), indent=1)
print(responseJson.replace("{", "").replace("}", ""))

The RTO API at backend uses Python Flask and to database to retrieve the information of the vehicle.

Here is the rto-api.py code

'''
Install all these packages with pip
Flask==1.1.2
Flask-RESTful==0.3.8
Flask-SQLAlchemy==2.4.3
Jinja2==2.11.2
SQLAlchemy==1.3.18
'''
from flask import Flask
from flask_restful import  Api, Resource, reqparse, abort, fields, marshal_with
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
#Restful
api = Api(app)
# Creating/loading sqlite data base. Can be treated as fake database fo RTO
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///rto.db'
db = SQLAlchemy(app)
# Adding the columns to the model.
class RtoModel(db.Model):
    Registration = db.Column(db.String(64), primary_key=True)
    Owner = db.Column(db.String(64), nullable=False)
    Maker = db.Column(db.String(64), nullable=False)
    Vehicle = db.Column(db.String(64), nullable=False)
    Fuel_Type = db.Column(db.String(64), nullable=False)
    Chassis = db.Column(db.String(64), nullable=False)
    Engine_Number = db.Column(db.String(64), nullable=False)
    Registration_Date = db.Column(db.String(64), nullable=False)
    Insurance_Valid_Upto = db.Column(db.String(64), nullable=False)
    def __repr__(self):
        return f"RTO( Registration={Registration}, Owner={Owner}, Maker={Maker}, Vehicle={Vehicle}, Fuel_Type={Fuel_Type}, Chassis={Chassis}, Engine_Number={Engine_Number}, Registration_Date={Registration_Date}, Insurance_Valid_Upto={Insurance_Valid_Upto})"

# do the below thing only once for creating the table. 
#db.create_all()
video_put_args = reqparse.RequestParser()
video_put_args.add_argument('Registration', type=str, help='Registration number is required', required=True)
video_put_args.add_argument('Owner', type=str, help="Owner's Name is required", required=True)
video_put_args.add_argument('Maker', type=str, help='Maker details are required', required=True)
video_put_args.add_argument('Vehicle', type=str, help='Vehicle type is required', required=True)
video_put_args.add_argument('Fuel_Type', type=str, help='Type of Fuel is required', required=True)
video_put_args.add_argument('Chassis', type=str, help='Chassis Number is required', required=True)
video_put_args.add_argument('Engine_Number', type=str, help='Engine Number is required', required=True)
video_put_args.add_argument('Registration_Date', type=str, help='Registration Date is required', required=True)
video_put_args.add_argument('Insurance_Valid_Upto', type=str, help='Insurance Due Date is required', required=True)

Resource_fields = {
    'Registration' : fields.String,
    'Owner': fields.String,
    'Maker': fields.String,
    'Vehicle': fields.String,
    'Fuel_Type': fields.String,
    'Chassis': fields.String,
    'Engine_Number': fields.String,
    'Registration_Date': fields.String,
    'Insurance_Valid_Upto': fields.String
}

class RTO(Resource):
    @marshal_with(Resource_fields)
    def get(self, vehicle_number):
        result = RtoModel.query.filter_by(Registration=vehicle_number).first()
        if not result:
            abort(404, message="Vahicle details Not found...")
        return result
    @marshal_with(Resource_fields)
    def put(self, vehicle_number):
        args = video_put_args.parse_args()
        result = RtoModel.query.filter_by(Registration=vehicle_number).first()
        if result:
            abort(409, message="Vehicle already exists...")
        vehicle = RtoModel(Registration=vehicle_number, Owner=args['Owner'], Maker=args['Maker'], Vehicle=args['Vehicle'], Fuel_Type=args['Fuel_Type'], Chassis=args['Chassis'], Engine_Number=args['Engine_Number'], Registration_Date=args['Registration_Date'], Insurance_Valid_Upto=args['Insurance_Valid_Upto'])
        db.session.add(vehicle)
        db.session.commit()
        return vehicle , 201
    def delete(self, video_id):
        return "", 204
api.add_resource(RTO, "/vehicle/<string:vehicle_number>")
if __name__ == '__main__':
    app.run(debug=True)

Now let’s look at the python code which fetches the associated vehicle info from this API

import requests
import json
BASE = 'http://127.0.0.1:5000/'

data = [{'Registration': 'KA05NB1786', 
        'Owner': 'WASIM AHMAD', 
        'Maker': 'SELTOS/ KIS', 
        'Vehicle': 'MOTOL CAR ', 
        'Fuel_Type': 'DEIESEL', 
        'Chassis': 'MZBEN813LLN16XXXXX', 
        'Engine_Number': 'D4FALM08XXXXX', 
        'Registration_Date': '23-10-2020', 
        'Insurance_Valid_Upto': '21-OCT-2023'
    }
]
for i in range(len(data)):
    response = requests.put(BASE + "vehicle/" + data[i]['Registration'], data[i])
    print(response.json())
input()
response = requests.get(BASE + "vehicle/"+ data[0]['Registration'])
print(json.dumps(response.json(), indent=1))

This is how with the integration of multiple concepts e-challans and lot more applications can be built.

Thank you! keep learning! keep growing! keep sharing!

Krithika Sharma
If you enjoyed this give it a clap, follow me on Medium for more
Let’s connect on LinkedIn