From c6d6698b7f2174c3bc27ac63ad40c2af8ddac818 Mon Sep 17 00:00:00 2001 From: Rahuldeb5 Date: Mon, 11 Mar 2024 18:37:20 -0400 Subject: [PATCH 1/5] Started documentation for Python Vision Code --- documentation.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 documentation.md diff --git a/documentation.md b/documentation.md new file mode 100644 index 0000000..e69de29 From 0c547d9d33b2739cc51789253fd8147b91a81c24 Mon Sep 17 00:00:00 2001 From: Rahuldeb75 Date: Mon, 11 Mar 2024 18:49:28 -0400 Subject: [PATCH 2/5] Started documentation for Python vision --- documentation.md | 0 documentation.txt | 89 +++++++++++++++++++++++++++++++++++++++++ src/output/Publisher.py | 2 +- 3 files changed, 90 insertions(+), 1 deletion(-) delete mode 100644 documentation.md create mode 100644 documentation.txt diff --git a/documentation.md b/documentation.md deleted file mode 100644 index e69de29..0000000 diff --git a/documentation.txt b/documentation.txt new file mode 100644 index 0000000..72e61f6 --- /dev/null +++ b/documentation.txt @@ -0,0 +1,89 @@ +Documentation by Rahul Deb + +Files to look at: +src/output/Publisher.py +src/__init__.py +src/pipeline/ + +src/output/Publisher.py + +9:15 - Imports +1) ntcore passes non-Driver station data to and from the robot across a network +https://github.com/robotpy/pyntcore/tree/main +2) Union represents objects that can be combined to obtain an ndarray +https://numpy.org/devdocs/reference/typing.html +3) Logging provides data as the code is running +https://docs.python.org/3/library/logging.html +4) Config allows us to connect to the camera and other information locally or remotely. +- LocalConfig states misallenous things + defines the camera matrix (mapping between 3D world and 2D image)and the distortion coefficient (used to correct the distortion). +- RemoteConfig provides information/settings about the camera. + +Self-note: Fiducial are AprilTags + +19:29 - +We define our functions and if they aren't implemented, we raise an exception + +31:39 - We convert the Pose3D to an array by getting the 3D translation and rotation values + +43:53 - Publishes data to the NetworkTable. +Creating and publishing data: +https://docs.wpilib.org/en/stable/docs/software/networktables/publish-and-subscribe.html +Information published: +- Frames per second +- Latency +- Translation vectors +- Rotation vectors +- the Ids +- The areas +- Checking for projection error + +All of this information is transferred together to the network (robot) -- a cloud + +55:62 (__init__ method) - Initializes the NetworkTable to default. It sets information about the server Ip, the client, creates the table, and starts logging. + +64:77 - Publishes information about fps, latency, tids, pose, areas, reprojection error, etc. +It publishes information periodically and sets most items to their default value. + +81:89 (send method) - +Takes in parameters of previously published items. Note: fps and latency are in a union with None, so a null value is possible for them. + +- If fps has a value, then set the publisher value to that +- + +??? 86:93 + +- Else set the other item values to empty arrays. + +100:101 (sendMsg method) publishes the message (string) + +103:108 - Closes all publisher values to reset it. + +-------------------------------------- +src/__init__.py + +9:18 - Imports + +Imports the OpenCV and the time modules. +https://docs.opencv.org/4.x/index.html +OpenCV is used for image processing. +https://docs.python.org/3/library/time.html + +Importing config, localConfig, remoteConfig (explained before). + +FileConfigManager configures the file where the data is logged. + +NTConfigManager configures/intializes the cameras in the NetworkTable. + +Annotate Fiducials. +The Annotate class views the current image and draws cubes based on the given April Tag and maps the unique features (cubes) of the AprilTag It uses OpenCV to draw contours in order to draw up a shape. + +MJPG Server loads a website (HTML file) with information related to the image content. + +The NetworkPublisher is imported + +Part of Pipeline: +1) The DefaultCapture is just information for the camera to run. + +2) FiducialDetector detects when a AprilTag is in sight. + +3) FiducialPoseEstimator ??? diff --git a/src/output/Publisher.py b/src/output/Publisher.py index cd9c333..71fbbb2 100644 --- a/src/output/Publisher.py +++ b/src/output/Publisher.py @@ -47,7 +47,7 @@ class NTPublisher: tids_pub: ntcore.IntegerArrayPublisher areas_pub: ntcore.DoubleArrayPublisher reprojection_error_pub: ntcore.DoublePublisher - + msg_pub: ntcore.StringPublisher update_counter_pub: ntcore.IntegerPublisher counter: int From c587879a317a15ea21cf951a927cd50f1936fe54 Mon Sep 17 00:00:00 2001 From: Rahuldeb75 Date: Tue, 12 Mar 2024 17:21:24 -0400 Subject: [PATCH 3/5] Updated documentation for Publisher.py --- src/output/Publisher.py | 46 ++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/src/output/Publisher.py b/src/output/Publisher.py index 71fbbb2..3b7ca6f 100644 --- a/src/output/Publisher.py +++ b/src/output/Publisher.py @@ -6,15 +6,28 @@ https://opensource.org/license/MIT. """ +""" +ntcore passes non-driver station data to and from the robot across a network +https://github.com/robotpy/pyntcore/tree/main +""" import ntcore from typing import Union import numpy.typing import logging from wpimath.geometry import * +""" +Config allows us to connect to the camera and other information locally or remotely. +- LocalConfig sets up misallenous information +- RemoteConfig sets up information about the camera +""" from config.Config import Config class Publisher: + """ + Initial definition of the functions. + Raises exceptions if the functions are not implmented + """ def __init__(self): raise NotImplementedError @@ -29,6 +42,7 @@ def close(self): raise NotImplementedError def poseToArray(pose: Pose3d): + #Converts the Pose3D into an array by mapping the XYZ translation and rotation values. return [ pose.translation().X(), pose.translation().Y(), @@ -39,12 +53,16 @@ def poseToArray(pose: Pose3d): ] class NTPublisher: - - fps_pub: ntcore.FloatPublisher - latency_pub: ntcore.DoublePublisher - tvecs_pub: ntcore.DoubleArrayPublisher - rvecs_pub: ntcore.DoubleArrayPublisher - tids_pub: ntcore.IntegerArrayPublisher + """ + Creates topics (data channels) and publishes values to it + https://docs.wpilib.org/en/stable/docs/software/networktables/publish-and-subscribe.html + """ + + fps_pub: ntcore.FloatPublisher #Frames per second + latency_pub: ntcore.DoublePublisher #Latency -- Time for data packet to transfer + tvecs_pub: ntcore.DoubleArrayPublisher #Translation vectors + rvecs_pub: ntcore.DoubleArrayPublisher #Rotation vectors + tids_pub: ntcore.IntegerArrayPublisher areas_pub: ntcore.DoubleArrayPublisher reprojection_error_pub: ntcore.DoublePublisher @@ -53,7 +71,11 @@ class NTPublisher: counter: int def __init__(self, config: Config): - + """ + The self parameter invokes the NetworkTable. + The config parameter provides information related to the camera. + """ + # Initializes an instance of a NetworkTable, gets misallenous information, and starts logging instance = ntcore.NetworkTableInstance.getDefault() instance.setServerTeam(config.local.team_number) instance.setServer(config.local.server_ip, ntcore.NetworkTableInstance.kDefaultPort4) @@ -61,6 +83,7 @@ def __init__(self, config: Config): table = instance.getTable("/" + config.local.device_name + "/output") logging.basicConfig(level=logging.DEBUG) + # Starts publishing data periodically to the NetworkTable and setting items to their default values self.fps_pub = table.getFloatTopic("fps").publish() self.fps_pub.setDefault(0) self.latency_pub = table.getDoubleTopic("latency").publish(ntcore.PubSubOptions(keepDuplicates=True, periodic=0.02)) @@ -78,11 +101,16 @@ def __init__(self, config: Config): self.counter = 0 - def send(self, fps: Union[float, None], latency: Union[float, None], tids, primary_pose, areas, reprojection_error): + def send(self, fps: Union[float, None], latency: Union[float, None], tids: Union[list, None], primary_pose: Union[list, None], areas: list, reprojection_error: Union[float, None]): + """ + Updates the data by setting their value if there's not nothing (aka theres's something). + Else, it sets values to default (empty array or 0) + """ if fps is not None: self.fps_pub.set(fps) + #ntcore._now() returns the current time (used to show when information was updated) if latency is not None and tids is not None and primary_pose is not None and len(areas) > 0: self.latency_pub.set(latency * 1000, ntcore._now()) self.tids_pub.set(tids, ntcore._now()) @@ -98,9 +126,11 @@ def send(self, fps: Union[float, None], latency: Union[float, None], tids, prima self.reprojection_error_pub.set(0) def sendMsg(self, msg: str): + #sends message (string) to the NetworkTable self.msg_pub.set(msg) def close(self): + #Stops publishing to the NetworkTable self.fps_pub.close() self.tids_pub.close() self.pose_sub.close() From c6c614f0f226f28d76f08de01c6a1850c976fc2b Mon Sep 17 00:00:00 2001 From: Rahuldeb75 Date: Tue, 12 Mar 2024 18:12:41 -0400 Subject: [PATCH 4/5] Updated documentation for Annotate.py --- src/output/Annotate.py | 44 +++++++++++++++++++++++++++++++++++++---- src/output/Publisher.py | 2 +- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/output/Annotate.py b/src/output/Annotate.py index f846dc8..acee1b5 100644 --- a/src/output/Annotate.py +++ b/src/output/Annotate.py @@ -6,32 +6,60 @@ https://opensource.org/license/MIT. """ +""" +cv2 library provides useful functions and methods for working with images +numpy provides a wide variety of mathematical operations on arrays +""" import cv2 import numpy +""" +Config allows us to connect to the camera and other information locally or remotely. +- LocalConfig sets up misallenous information +- RemoteConfig sets up information about the camera +""" from config.Config import Config class Annotate: + """ + Initial definition of the functions. + Raises exceptions if the functions are not implmented + """ + def __init__(self): raise NotImplementedError def annotate(self, image, config: Config): raise NotImplementedError - + @classmethod - def drawCube(self, frame, image_points): + def drawCube(self, frame, image_points:list): + """ + The frame parameter provides the initial image, where cv2 will perform operations on + The image_points parameter is a list containing the points in the image to perform operations on. + drawCube(...) draws contours and lines based on image_points and returns the edited image + """ + + #Reshapes the array to a 2D array where the 2nd dimension has a size of 2 image_points = numpy.int32(image_points).reshape(-1,2) + #cv2 draws contours by extracting the image points and draws ALL the contours. + #contours are outlines/boundaries of objects in an image frame = cv2.drawContours(frame, [image_points[:4]], -1, (0,255,0), 3) + #runs a nested for loop to draw the lines between every pair of image points (given 2 points, exactly one line connects them both) for i, j in zip(range(4),range(4,8)): frame = cv2.line(frame, tuple(image_points[i]), tuple(image_points[j]), (255), 3) - + frame = cv2.drawContours(frame, [image_points[4:]], -1, (0,0,255), 3) return frame class AnnotateFiducials(Annotate): + """ + Annotates the AprilTags using the class previously defined. + Returns the annotated AprilTag image + """ fiducial_size = 0.0 axis = numpy.array([]) @@ -41,13 +69,21 @@ def __init__(self): def annotate(self, image, rvecs, tvecs, fps, fpt, config: Config): if config.remote.fiducial_size != self.fiducial_size: - self.fiducial_size = config.remote.fiducial_size + self.fiducial_size = config.remote.fiducial_size #sets the correct AprilTag size based on information from Config + #Sets the 2nd dimension arrays to the maximum boundaries (the outer edges of the AprilTag) self.axis = numpy.float32([[-self.fiducial_size/2,-self.fiducial_size/2,0], [self.fiducial_size/2,-self.fiducial_size/2,0], [self.fiducial_size/2,self.fiducial_size/2,0], [-self.fiducial_size/2,self.fiducial_size/2,0], [-self.fiducial_size/2,-self.fiducial_size/2,self.fiducial_size],[self.fiducial_size/2,-self.fiducial_size/2,self.fiducial_size], [self.fiducial_size/2,self.fiducial_size/2,self.fiducial_size],[-self.fiducial_size/2,self.fiducial_size/2,self.fiducial_size] ]) + """ + Nested for loop through the rotation and translation vectors. + The rotation vetors are converted to a Rodrigues rotation vector (compact representation of a 3D rotation), which is more efficient and easier to manipulate + The 3D image points are projected into a 2D image plane + The new 2D image plane is annotated + The annotated 2D image is returned. + """ for rvec, tvec in zip(rvecs, tvecs): rvec, _ = cv2.Rodrigues(rvec) image_points, _ = cv2.projectPoints(self.axis, rvec, tvec, config.local.camera_matrix, config.local.distortion_coefficient) diff --git a/src/output/Publisher.py b/src/output/Publisher.py index 3b7ca6f..3cbe78a 100644 --- a/src/output/Publisher.py +++ b/src/output/Publisher.py @@ -28,7 +28,7 @@ class Publisher: Initial definition of the functions. Raises exceptions if the functions are not implmented """ - + def __init__(self): raise NotImplementedError From a2090c57e7b8d8dee8dc86729eb5660598932f5d Mon Sep 17 00:00:00 2001 From: Rahuldeb75 Date: Tue, 12 Mar 2024 18:37:38 -0400 Subject: [PATCH 5/5] Updated __init__.py and deleted documentation.txt --- documentation.txt | 89 ----------------------------------------- src/__init__.py | 24 ++++++++++- src/output/Annotate.py | 5 --- src/output/Publisher.py | 8 +--- 4 files changed, 25 insertions(+), 101 deletions(-) delete mode 100644 documentation.txt diff --git a/documentation.txt b/documentation.txt deleted file mode 100644 index 72e61f6..0000000 --- a/documentation.txt +++ /dev/null @@ -1,89 +0,0 @@ -Documentation by Rahul Deb - -Files to look at: -src/output/Publisher.py -src/__init__.py -src/pipeline/ - -src/output/Publisher.py - -9:15 - Imports -1) ntcore passes non-Driver station data to and from the robot across a network -https://github.com/robotpy/pyntcore/tree/main -2) Union represents objects that can be combined to obtain an ndarray -https://numpy.org/devdocs/reference/typing.html -3) Logging provides data as the code is running -https://docs.python.org/3/library/logging.html -4) Config allows us to connect to the camera and other information locally or remotely. -- LocalConfig states misallenous things + defines the camera matrix (mapping between 3D world and 2D image)and the distortion coefficient (used to correct the distortion). -- RemoteConfig provides information/settings about the camera. - -Self-note: Fiducial are AprilTags - -19:29 - -We define our functions and if they aren't implemented, we raise an exception - -31:39 - We convert the Pose3D to an array by getting the 3D translation and rotation values - -43:53 - Publishes data to the NetworkTable. -Creating and publishing data: -https://docs.wpilib.org/en/stable/docs/software/networktables/publish-and-subscribe.html -Information published: -- Frames per second -- Latency -- Translation vectors -- Rotation vectors -- the Ids -- The areas -- Checking for projection error - -All of this information is transferred together to the network (robot) -- a cloud - -55:62 (__init__ method) - Initializes the NetworkTable to default. It sets information about the server Ip, the client, creates the table, and starts logging. - -64:77 - Publishes information about fps, latency, tids, pose, areas, reprojection error, etc. -It publishes information periodically and sets most items to their default value. - -81:89 (send method) - -Takes in parameters of previously published items. Note: fps and latency are in a union with None, so a null value is possible for them. - -- If fps has a value, then set the publisher value to that -- - -??? 86:93 - -- Else set the other item values to empty arrays. - -100:101 (sendMsg method) publishes the message (string) - -103:108 - Closes all publisher values to reset it. - --------------------------------------- -src/__init__.py - -9:18 - Imports - -Imports the OpenCV and the time modules. -https://docs.opencv.org/4.x/index.html -OpenCV is used for image processing. -https://docs.python.org/3/library/time.html - -Importing config, localConfig, remoteConfig (explained before). - -FileConfigManager configures the file where the data is logged. - -NTConfigManager configures/intializes the cameras in the NetworkTable. - -Annotate Fiducials. -The Annotate class views the current image and draws cubes based on the given April Tag and maps the unique features (cubes) of the AprilTag It uses OpenCV to draw contours in order to draw up a shape. - -MJPG Server loads a website (HTML file) with information related to the image content. - -The NetworkPublisher is imported - -Part of Pipeline: -1) The DefaultCapture is just information for the camera to run. - -2) FiducialDetector detects when a AprilTag is in sight. - -3) FiducialPoseEstimator ??? diff --git a/src/__init__.py b/src/__init__.py index 888516d..9669ede 100755 --- a/src/__init__.py +++ b/src/__init__.py @@ -6,6 +6,9 @@ https://opensource.org/license/MIT. """ +""" +cv2 library provides useful functions and methods for working with images +""" import time, cv2 from config.Config import Config, LocalConfig, RemoteConfig @@ -17,6 +20,15 @@ from pipeline.Detector import FiducialDetector from pipeline.PoseEstimator import FiducialPoseEstimator +""" +Config allows us to connect to the camera and other information locally or remotely. +- LocalConfig sets up misallenous information +- RemoteConfig sets up information about the camera + +FileConfigManager() configures the path, IP, and provides other miscellaneous information + +NTConfigManager configures the NetworkTable by setting the subscribers and topics +""" config = Config(LocalConfig(), RemoteConfig()) file_config_manager = FileConfigManager() nt_config_manager = NTConfigManager() @@ -32,6 +44,9 @@ capture = DefaultCapture(publisher) def main(): + """ + Connects the vision information with other classes throughout the repository + """ publisher.sendMsg(config.local.device_name + " has started") @@ -49,15 +64,21 @@ def main(): frame = capture.getFrame(config) if frame is None: + # Handling error if frame doesn't exist publisher.sendMsg("Camera not connected") publisher.send(0, 0, None, None) capture.release() continue - + + # extracting information from the frame fiducials, tids, all_corners = detector.detect(frame) areas = [] if tids is not None and all_corners is not None: + """ + Detects the ARUCO markers from the image + https://docs.opencv.org/4.x/d5/dae/tutorial_aruco_detection.html + """ frame = cv2.aruco.drawDetectedMarkers(frame, all_corners, tids) tids, areas = detector.orderIDs(all_corners, tids) @@ -77,5 +98,6 @@ def main(): try: main() except KeyboardInterrupt: + # resets the camera and publisher capture.release() publisher.close() diff --git a/src/output/Annotate.py b/src/output/Annotate.py index acee1b5..85890e9 100644 --- a/src/output/Annotate.py +++ b/src/output/Annotate.py @@ -13,11 +13,6 @@ import cv2 import numpy -""" -Config allows us to connect to the camera and other information locally or remotely. -- LocalConfig sets up misallenous information -- RemoteConfig sets up information about the camera -""" from config.Config import Config class Annotate: diff --git a/src/output/Publisher.py b/src/output/Publisher.py index 3cbe78a..fdea55c 100644 --- a/src/output/Publisher.py +++ b/src/output/Publisher.py @@ -16,11 +16,7 @@ import logging from wpimath.geometry import * -""" -Config allows us to connect to the camera and other information locally or remotely. -- LocalConfig sets up misallenous information -- RemoteConfig sets up information about the camera -""" + from config.Config import Config class Publisher: @@ -28,7 +24,7 @@ class Publisher: Initial definition of the functions. Raises exceptions if the functions are not implmented """ - + def __init__(self): raise NotImplementedError