Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OCR on Old Standard iFit Version #110

Open
wants to merge 25 commits into
base: master
Choose a base branch
from
Open

OCR on Old Standard iFit Version #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
93df4cb
Update ScreenCaptureService.java
cagnulein Sep 13, 2024
fd0e51a
Update QZService.java
cagnulein Sep 13, 2024
98dfb01
Update main.yml
cagnulein Sep 13, 2024
b673bd6
Update QZService.java
cagnulein Sep 13, 2024
d498610
Update QZService.java
cagnulein Sep 16, 2024
76deef1
Merge branch 'master' into ocr_standard_ifit_version
cagnulein Sep 20, 2024
81c0525
pausing the parse engine leaving only the ocr one
cagnulein Sep 20, 2024
b3a939e
fixing dirty wattage values
cagnulein Sep 20, 2024
f87c040
fixing wattage cadence and resistance
cagnulein Sep 24, 2024
a3ca297
Merge branch 'master' into ocr_standard_ifit_version
cagnulein Nov 28, 2024
c799779
fixing merge
cagnulein Nov 30, 2024
6278b9d
trying to improve errors
cagnulein Nov 30, 2024
0702fde
Merge branch 'ocr' into ocr_standard_ifit_version
cagnulein Nov 30, 2024
b495dbe
Update QZService.java
cagnulein Dec 5, 2024
9fede38
Merge branch 'ocr' into ocr_standard_ifit_version
cagnulein Dec 5, 2024
5099b18
sending ocr lines over udp to debug them
cagnulein Dec 9, 2024
0dfbed3
fixing socket open for sendbroadcast
cagnulein Dec 9, 2024
e62faa4
Update QZService.java
cagnulein Dec 9, 2024
a2c3d68
fixing build
cagnulein Dec 9, 2024
6214293
cadence filter by ocr
cagnulein Dec 18, 2024
689df1b
debuglog setting
cagnulein Dec 12, 2024
a4bd9e2
Merge branch 'ocr' into ocr_standard_ifit_version
cagnulein Dec 18, 2024
bd939c9
Update QZService.java
cagnulein Dec 18, 2024
c0f60a6
Update QZService.java
cagnulein Dec 18, 2024
808c948
Filtering strange wattage reading
cagnulein Dec 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions app/src/main/java/org/cagnulein/qzcompanionnordictracktreadmill/MainActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,11 @@ public void run() {
}

startOCR();

// With OCR we can't move it to back here, only after asking the permission, restore it somewhere
/*if (savedInstanceState == null) { // Verifica se è la prima creazione
moveTaskToBack(true); // Sposta l'Activity in background
}*/
}

private boolean isAccessibilityServiceEnabled(Context context, Class<?> accessibilityService) {
Expand Down
21 changes: 12 additions & 9 deletions app/src/main/java/org/cagnulein/qzcompanionnordictracktreadmill/QZService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,11 +65,9 @@ public void onCreate() {
sharedPreferences = getSharedPreferences("QZ",MODE_PRIVATE);
try {
broadcastAddress = getBroadcastAddress();
} catch (IOException e) {
Log.e(LOG_TAG, e.getMessage());
} catch (Exception e) {
Log.e(LOG_TAG, e.toString());
}

// The service is being created
//Toast.makeText(this, "Service created!", Toast.LENGTH_LONG).show();
writeLog( "Service onCreate");

Expand All @@ -78,7 +76,6 @@ public void onCreate() {
@Override
public void run() {
writeLog( "Service run"); /*parse();*/ getOCR();
handler.postDelayed(runnable, 100);
}
};
} finally {
Expand Down Expand Up @@ -211,12 +208,14 @@ public String[] getOCR() {
}

}

if (lines[i].toLowerCase().contains("cadence") || lines[i].toLowerCase().contains("rpm")) {
try {
String potentialNumber = lines[i-1].trim();
// Try to parse the number to check if it's valid
Double.parseDouble(potentialNumber);
QZService.lastCadence = "Changed RPM " + potentialNumber;
Double dd = Double.parseDouble(potentialNumber);
if(dd > 30.0 && dd < 120.0)
QZService.lastCadence = "Changed RPM " + potentialNumber;
} catch (Exception e) {
// If lines[i-1] isn't a number, try lines[i-2]
try {
Expand Down Expand Up @@ -246,7 +245,9 @@ public String[] getOCR() {
String[] numbers = numberStr.split("\\s+");
if (numbers.length > 0) {
int watts = Integer.parseInt(numbers[numbers.length - 1]);
QZService.lastWattage = "Changed Watts " + watts;
if(watts > 20) { // filtering wrong reading from OCR
QZService.lastWattage = "Changed Watts " + watts;
}
}
} catch (Exception e) {
}
Expand Down Expand Up @@ -277,6 +278,7 @@ public String[] getOCR() {
ex.printStackTrace();
return result;
}
handler.postDelayed(runnable, 100);
return result;
}

Expand Down Expand Up @@ -533,11 +535,12 @@ public static void sendBroadcast(String messageStr) {
byte[] sendData = messageStr.getBytes();
DatagramPacket sendPacket = new DatagramPacket(sendData, sendData.length, broadcastAddress, clientPort);
socket.send(sendPacket);

socket.close();
} catch (IOException e) {
Log.e(LOG_TAG, "IOException: " + e.getMessage());
}


}

InetAddress getBroadcastAddress() throws IOException {
Expand Down
173 changes: 125 additions & 48 deletions app/src/main/java/org/cagnulein/qzcompanionnordictracktreadmill/ScreenCaptureService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.nio.ByteBuffer;
import java.util.Objects;

import com.google.android.gms.tasks.OnCompleteListener;
import com.google.mlkit.vision.common.InputImage;
import com.google.mlkit.vision.text.Text;
import com.google.mlkit.vision.text.TextRecognition;
Expand All @@ -42,6 +43,7 @@
import android.graphics.Rect;
import android.graphics.Point;

import androidx.annotation.NonNull;
import androidx.core.util.Pair;
import android.util.Log;
import android.os.Build;
Expand Down Expand Up @@ -125,72 +127,147 @@ private static int getVirtualDisplayFlags() {
}

private class ImageAvailableListener implements ImageReader.OnImageAvailableListener {
private static final String TAG = "OCR";
private static final int PROCESSING_TIMEOUT_MS = 5000;
private final Handler timeoutHandler = new Handler();

private class BitmapHolder {
Bitmap fullBitmap;
Bitmap roiBitmap;

void recycleBitmaps() {
if (fullBitmap != null && !fullBitmap.isRecycled()) {
fullBitmap.recycle();
Log.d(TAG, "Full bitmap recycled");
}
if (roiBitmap != null && !roiBitmap.isRecycled()) {
roiBitmap.recycle();
Log.d(TAG, "ROI bitmap recycled");
}
}
}

@Override
public void onImageAvailable(ImageReader reader) {
try (Image image = mImageReader.acquireLatestImage()) {
if (image != null && !isRunning) {
Log.i("OCR","running");
Image.Plane[] planes = image.getPlanes();
ByteBuffer buffer = planes[0].getBuffer();
int pixelStride = planes[0].getPixelStride();
int rowStride = planes[0].getRowStride();
int rowPadding = rowStride - pixelStride * mWidth;

isRunning = true;

// Create full bitmap
int fullWidth = mWidth + rowPadding / pixelStride;
int fullHeight = mHeight;
Bitmap fullBitmap = Bitmap.createBitmap(fullWidth, fullHeight, Bitmap.Config.ARGB_8888);
fullBitmap.copyPixelsFromBuffer(buffer);

// Calculate the region of interest (last 100 pixels)
//int roiHeight = Math.min(100, fullHeight);
int roiHeight = fullHeight;
int roiY = fullHeight - roiHeight;

// Create a new bitmap for the region of interest
Bitmap roiBitmap = Bitmap.createBitmap(fullBitmap, 0, roiY, fullWidth, roiHeight);

// Recycle the full bitmap as we no longer need it
fullBitmap.recycle();
Log.d(TAG, "onImageAvailable called");

// If already processing, acquire and discard the image
if (isRunning) {
Log.d(TAG, "System busy - acquiring and discarding latest image");
Image skippedImage = mImageReader.acquireLatestImage();
if (skippedImage != null) {
skippedImage.close();
Log.d(TAG, "Skipped image closed");
}
return;
}

final BitmapHolder bitmapHolder = new BitmapHolder();
Image image = null;

// Use roiBitmap for OCR
InputImage inputImage = InputImage.fromBitmap(roiBitmap, 0);
try {
// Acquire the most recent image
image = mImageReader.acquireLatestImage();
if (image == null) {
Log.e(TAG, "Acquired image is null");
return;
}

isRunning = true;
Log.d(TAG, "Starting image processing");

// Set timeout
timeoutHandler.postDelayed(new Runnable() {
@Override
public void run() {
if (isRunning) {
isRunning = false;
bitmapHolder.recycleBitmaps();
Log.w(TAG, "Processing timeout reached - forcing reset");
}
}
}, PROCESSING_TIMEOUT_MS);

Task<Text> result = recognizer.process(inputImage)
// Get image data
Image.Plane[] planes = image.getPlanes();
ByteBuffer buffer = planes[0].getBuffer();
int pixelStride = planes[0].getPixelStride();
int rowStride = planes[0].getRowStride();
int rowPadding = rowStride - pixelStride * mWidth;

// Create full bitmap
int fullWidth = mWidth + rowPadding / pixelStride;
int fullHeight = mHeight;
Log.d(TAG, "Creating bitmap with dimensions: " + fullWidth + "x" + fullHeight);

bitmapHolder.fullBitmap = Bitmap.createBitmap(fullWidth, fullHeight, Bitmap.Config.ARGB_8888);
bitmapHolder.fullBitmap.copyPixelsFromBuffer(buffer);

// Create ROI
int roiHeight = Math.min(300, fullHeight);
Log.d(TAG, "Creating ROI bitmap with height: " + roiHeight);

bitmapHolder.roiBitmap = Bitmap.createBitmap(bitmapHolder.fullBitmap, 0, 0, fullWidth, roiHeight);
bitmapHolder.fullBitmap.recycle();
Log.d(TAG, "Full bitmap recycled after ROI creation");

// Process OCR
InputImage inputImage = InputImage.fromBitmap(bitmapHolder.roiBitmap, 0);
Log.d(TAG, "Starting OCR processing");

recognizer.process(inputImage)
.addOnSuccessListener(new OnSuccessListener<Text>() {
@Override
public void onSuccess(Text result) {
Log.i("OCR","processed");
// Process OCR result as before
String resultText = result.getText();
lastText = resultText;
lastTextExtended = "";
for (Text.TextBlock block : result.getTextBlocks()) {
String blockText = block.getText();
Log.i("OCR",blockText);
Rect blockFrame = block.getBoundingBox();
// Adjust the Y coordinate of the bounding box
if (blockFrame != null) {
blockFrame.offset(0, roiY);
Log.d(TAG, "OCR processing successful");
try {
String resultText = result.getText();
lastText = resultText;
lastTextExtended = "";

for (Text.TextBlock block : result.getTextBlocks()) {
String blockText = block.getText();
Rect blockFrame = block.getBoundingBox();
if (blockFrame != null) {
blockFrame.offset(0, 0);
}
lastTextExtended += blockText + "$$" + blockFrame.toString() + "§§";
}
lastTextExtended += blockText + "$$" + blockFrame.toString() + "§§";

Log.d(TAG, "Processed text length: " + resultText.length());
Log.d(TAG, "Number of text blocks: " + result.getTextBlocks().size());
} catch (Exception e) {
Log.e(TAG, "Error processing OCR result", e);
}
roiBitmap.recycle();
isRunning = false;
}
})
.addOnFailureListener(new OnFailureListener() {
@Override
public void onFailure(Exception e) {
public void onFailure(@NonNull Exception e) {
Log.e(TAG, "OCR processing failed", e);
}
})
.addOnCompleteListener(new OnCompleteListener<Text>() {
@Override
public void onComplete(@NonNull Task<Text> task) {
Log.d(TAG, "OCR task completed");
timeoutHandler.removeCallbacksAndMessages(null);
isRunning = false;
bitmapHolder.recycleBitmaps();
}
});
}

} catch (Exception e) {
e.printStackTrace();
Log.e(TAG, "Error in image processing", e);
isRunning = false;
if (bitmapHolder != null) {
bitmapHolder.recycleBitmaps();
}
} finally {
if (image != null) {
image.close();
Log.d(TAG, "Image closed");
}
}
}
}
Expand Down
Loading