run gui script

README.md

@@ -48,6 +48,23 @@ cd saw_mill_knot_detection
 python -m venv .venv
 source .venv/bin/activate
 
+# Install dependencies
+pip install -r requirements.txt
+```
+
+### 2. Run the Annotation GUI
+
+The repository includes an automated script that handles virtual environment activation:
+
+```bash
+# Run the GUI (automatically detects and activates venv/conda environment)
+./run_gui.sh
+
+# Or run manually
+source .venv/bin/activate  # or conda activate your_env
+python annotation_gui.py
+```
+
 # Install dependencies
 pip install -U pip
 pip install ultralytics gradio rfdetr

annotation_gui.py

@@ -328,14 +328,14 @@ class AnnotationApp:
         
         return f"❌ Model '{model_display}' not found"
     
-    def load_new_images_dir(self, images_dir: str) -> tuple[Image.Image | None, str, str]:
+    def load_new_images_dir(self, images_dir: str) -> tuple[str, str, str]:
         """Load a new images directory from the GUI."""
         path = Path(images_dir)
         if not path.exists():
-            return None, "", f"❌ Directory not found: {images_dir}"
+            return "<div>Directory not found</div>", "", f"❌ Directory not found: {images_dir}"
         
         if not path.is_dir():
-            return None, "", f"❌ Not a directory: {images_dir}"
+            return "<div>Not a directory</div>", "", f"❌ Not a directory: {images_dir}"
         
         result = self._load_images(path)
         
@@ -343,12 +343,12 @@ class AnnotationApp:
         if self.image_paths:
             img, filename = self.get_current_image()
             boxes = self.annotations.get(filename, [])
-            img_with_boxes = self.draw_boxes_on_image(img, boxes) if boxes else img
+            img_html = self.generate_interactive_canvas(boxes)
             boxes_text = self._format_boxes_text(boxes)
             info = f"{result}\nImage 1/{len(self.image_paths)}: {filename}"
-            return img_with_boxes, boxes_text, info
+            return img_html, boxes_text, info
         else:
-            return None, "", f"{result}\n⚠️ No .jpg or .png images found in directory"
+            return "<div>No images found</div>", "", f"{result}\n⚠️ No .jpg or .png images found in directory"
     
     def get_current_model_info(self) -> str:
         """Get info about currently loaded model."""
@@ -385,73 +385,285 @@ class AnnotationApp:
             # Draw box
             draw.rectangle([x1, y1, x2, y2], outline="red", width=3)
             
+            # Draw corner handles for editing (small squares)
+            handle_size = 6
+            draw.rectangle([x1-handle_size, y1-handle_size, x1+handle_size, y1+handle_size], fill="red")
+            draw.rectangle([x2-handle_size, y1-handle_size, x2+handle_size, y1+handle_size], fill="red")
+            draw.rectangle([x1-handle_size, y2-handle_size, x1+handle_size, y2+handle_size], fill="red")
+            draw.rectangle([x2-handle_size, y2-handle_size, x2+handle_size, y2+handle_size], fill="red")
+            
             # Draw label
             text = f"{label} {conf:.2f}" if conf < 1.0 else label
             draw.text((x1, y1 - 20), text, fill="red")
         
         return img_draw
     
-    def auto_label_current(self, threshold: float = 0.5) -> tuple[Image.Image, str, str]:
+    def generate_interactive_canvas(self, boxes: list[dict] = None) -> str:
-        """Auto-label current image using loaded model."""
+        """Generate HTML with interactive canvas for annotation."""
-        if not self.model:
-            return None, "", "❌ No model loaded"
-        
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No image loaded</div>"
         
-        try:
+        if boxes is None:
-            # Run inference based on model type
+            boxes = self.annotations.get(filename, [])
-            if self.current_model_type == "rf-detr":
+        
-                # RF-DETR custom prediction
+        # Resize image for display if too large
-                detections = self.model.predict(img, threshold=threshold)
+        max_width = 1200
-                boxes = []
+        max_height = 800
-                for i in range(len(detections)):
+        if img.width > max_width or img.height > max_height:
-                    xyxy = detections.xyxy[i]
+            ratio = min(max_width / img.width, max_height / img.height)
-                    conf = float(detections.confidence[i]) if detections.confidence is not None else 1.0
+            display_width = int(img.width * ratio)
-                    x1, y1, x2, y2 = xyxy
+            display_height = int(img.height * ratio)
-                    boxes.append({
+            img = img.resize((display_width, display_height), Image.Resampling.LANCZOS)
-                        "bbox": [float(x1), float(y1), float(x2), float(y2)],
-                        "label": "knot",
-                        "confidence": conf,
-                        "source": "auto"
-                    })
         else:
-                # Ultralytics models (RT-DETR, YOLOv6, YOLOX)
+            display_width = img.width
-                results = self.model.predict(
+            display_height = img.height
-                    source=img,
-                    conf=threshold,
-                    save=False,
-                    verbose=False
-                )
         
-                boxes = []
+        # Convert PIL image to base64
-                for result in results:
+        import base64
-                    for box in result.boxes:
+        from io import BytesIO
-                        x1, y1, x2, y2 = box.xyxy[0].tolist()
+        buffer = BytesIO()
-                        conf = float(box.conf[0])
+        img.save(buffer, format="PNG", optimize=True)
-                        boxes.append({
+        img_base64 = base64.b64encode(buffer.getvalue()).decode()
-                            "bbox": [x1, y1, x2, y2],
-                            "label": "knot",
-                            "confidence": conf,
-                            "source": "auto"
-                        })
         
-            # Add to existing annotations
+        # Generate HTML with canvas and JavaScript
-            if filename not in self.annotations:
+        boxes_json = json.dumps(boxes)
-                self.annotations[filename] = []
-            self.annotations[filename].extend(boxes)
-            self._save_annotations()
         
-            # Redraw
+        html = f"""
-            img_with_boxes = self.draw_boxes_on_image(img, self.annotations[filename])
+        <div style="position: relative; display: inline-block; border: 1px solid #ccc; padding: 5px;">
-            boxes_text = self._format_boxes_text(self.annotations[filename])
+            <div>Canvas Size: {display_width}x{display_height}</div>
-            info = f"🤖 Auto-labeled: {len(boxes)} detections | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
+            <canvas id="annotation-canvas" width="{display_width}" height="{display_height}" 
+                    style="border: 1px solid #ccc; cursor: crosshair; max-width: 100%; height: auto; background-color: #f0f0f0;"></canvas>
+            <div id="canvas-info" style="margin-top: 5px; font-size: 12px; color: #666;">
+                Canvas ready - testing image loading
+            </div>
+        </div>
+        <script>
+        (function() {{
+            const canvas = document.getElementById('annotation-canvas');
+            const ctx = canvas.getContext('2d');
+            const img = new Image();
+            img.src = 'data:image/png;base64,{img_base64}';
             
-            return img_with_boxes, boxes_text, info
+            let boxes = {boxes_json};
+            let isDragging = false;
+            let dragStart = null;
+            let selectedCorner = null;
+            let selectedBoxIndex = -1;
+            let creatingBox = false;
+            let createStart = null;
             
-        except Exception as e:
+            // Handle size for high DPI displays
-            return img, self._format_boxes_text(self.annotations.get(filename, [])), f"❌ Auto-label failed: {e}"
+            const devicePixelRatio = window.devicePixelRatio || 1;
+            const displayWidth = {display_width};
+            const displayHeight = {display_height};
+            // Temporarily disable high DPI scaling to fix image display
+            // canvas.width = displayWidth * devicePixelRatio;
+            // canvas.height = displayHeight * devicePixelRatio;
+            // canvas.style.width = displayWidth + 'px';
+            // canvas.style.height = displayHeight + 'px';
+            // ctx.scale(devicePixelRatio, devicePixelRatio);
+            
+            img.onload = function() {{
+                console.log('Image loaded successfully, size:', img.width, 'x', img.height);
+                redraw();
+            }};
+            
+            img.onerror = function() {{
+                console.error('Failed to load image, drawing test pattern');
+                // Draw a test pattern if image fails to load
+                ctx.fillStyle = '#ffcccc';
+                ctx.fillRect(0, 0, displayWidth, displayHeight);
+                ctx.fillStyle = 'black';
+                ctx.font = '20px Arial';
+                ctx.fillText('Image failed to load', 10, 30);
+                ctx.fillText('Canvas size: ' + displayWidth + 'x' + displayHeight, 10, 60);
+            }};
+            
+            // Force redraw after a short delay
+            setTimeout(function() {{
+                console.log('Timeout check - image complete:', img.complete, 'natural size:', img.naturalWidth, 'x', img.naturalHeight);
+                if (!img.complete || img.naturalWidth === 0) {{
+                    console.log('Image not loaded, drawing test pattern');
+                    ctx.fillStyle = '#ccccff';
+                    ctx.fillRect(0, 0, displayWidth, displayHeight);
+                    ctx.fillStyle = 'black';
+                    ctx.font = '16px Arial';
+                    ctx.fillText('Waiting for image...', 10, 30);
+                    ctx.fillText('Canvas: ' + displayWidth + 'x' + displayHeight, 10, 50);
+                }}
+            }}, 500);
+            
+            function redraw() {{
+                ctx.clearRect(0, 0, displayWidth, displayHeight);
+                ctx.drawImage(img, 0, 0, displayWidth, displayHeight);
+                
+                // Draw boxes
+                boxes.forEach((box, index) => {{
+                    const [x1, y1, x2, y2] = box.bbox;
+                    const label = box.label || 'knot';
+                    const conf = box.confidence || 1.0;
+                    
+                    // Draw box
+                    ctx.strokeStyle = 'red';
+                    ctx.lineWidth = 3;
+                    ctx.strokeRect(x1, y1, x2 - x1, y2 - y1);
+                    
+                    // Draw corner handles
+                    const handleSize = 6;
+                    ctx.fillStyle = 'red';
+                    ctx.fillRect(x1 - handleSize, y1 - handleSize, handleSize * 2, handleSize * 2);
+                    ctx.fillRect(x2 - handleSize, y1 - handleSize, handleSize * 2, handleSize * 2);
+                    ctx.fillRect(x1 - handleSize, y2 - handleSize, handleSize * 2, handleSize * 2);
+                    ctx.fillRect(x2 - handleSize, y2 - handleSize, handleSize * 2, handleSize * 2);
+                    
+                    // Draw label
+                    ctx.fillStyle = 'red';
+                    ctx.font = '16px Arial';
+                    const text = conf < 1.0 ? `${{label}} ${{conf.toFixed(2)}}` : label;
+                    ctx.fillText(text, x1, y1 - 5);
+                }});
+                
+                // Draw creation box if creating
+                if (creatingBox && createStart && dragStart) {{
+                    ctx.strokeStyle = 'blue';
+                    ctx.lineWidth = 2;
+                    ctx.setLineDash([5, 5]);
+                    const x = Math.min(createStart.x, dragStart.x);
+                    const y = Math.min(createStart.y, dragStart.y);
+                    const w = Math.abs(createStart.x - dragStart.x);
+                    const h = Math.abs(createStart.y - dragStart.y);
+                    ctx.strokeRect(x, y, w, h);
+                    ctx.setLineDash([]);
+                }}
+            }}
+            
+            function getCornerAt(x, y) {{
+                const handleSize = 6;
+                for (let i = 0; i < boxes.length; i++) {{
+                    const [x1, y1, x2, y2] = boxes[i].bbox;
+                    const corners = [
+                        {{x: x1, y: y1, type: 'top-left'}},
+                        {{x: x2, y: y1, type: 'top-right'}},
+                        {{x: x1, y: y2, type: 'bottom-left'}},
+                        {{x: x2, y: y2, type: 'bottom-right'}}
+                    ];
+                    
+                    for (let corner of corners) {{
+                        if (x >= corner.x - handleSize && x <= corner.x + handleSize &&
+                            y >= corner.y - handleSize && y <= corner.y + handleSize) {{
+                            return {{boxIndex: i, corner: corner.type, pos: corner}};
+                        }}
+                    }}
+                }}
+                return null;
+            }}
+            
+            canvas.addEventListener('mousedown', function(e) {{
+                const rect = canvas.getBoundingClientRect();
+                const x = (e.clientX - rect.left) * (displayWidth / rect.width);
+                const y = (e.clientY - rect.top) * (displayHeight / rect.height);
+                
+                selectedCorner = getCornerAt(x, y);
+                if (selectedCorner) {{
+                    isDragging = true;
+                    selectedBoxIndex = selectedCorner.boxIndex;
+                    canvas.style.cursor = 'move';
+                }} else {{
+                    // Start creating new box
+                    creatingBox = true;
+                    createStart = {{x: x, y: y}};
+                    dragStart = {{x: x, y: y}};
+                    canvas.style.cursor = 'crosshair';
+                }}
+            }});
+            
+            canvas.addEventListener('mousemove', function(e) {{
+                const rect = canvas.getBoundingClientRect();
+                const x = (e.clientX - rect.left) * (displayWidth / rect.width);
+                const y = (e.clientY - rect.top) * (displayHeight / rect.height);
+                
+                if (isDragging && selectedCorner) {{
+                    const box = boxes[selectedBoxIndex];
+                    if (selectedCorner.corner === 'top-left') {{
+                        box.bbox[0] = Math.min(x, box.bbox[2] - 10); // x1
+                        box.bbox[1] = Math.min(y, box.bbox[3] - 10); // y1
+                    }} else if (selectedCorner.corner === 'top-right') {{
+                        box.bbox[2] = Math.max(x, box.bbox[0] + 10); // x2
+                        box.bbox[1] = Math.min(y, box.bbox[3] - 10); // y1
+                    }} else if (selectedCorner.corner === 'bottom-left') {{
+                        box.bbox[0] = Math.min(x, box.bbox[2] - 10); // x1
+                        box.bbox[3] = Math.max(y, box.bbox[1] + 10); // y2
+                    }} else if (selectedCorner.corner === 'bottom-right') {{
+                        box.bbox[2] = Math.max(x, box.bbox[0] + 10); // x2
+                        box.bbox[3] = Math.max(y, box.bbox[1] + 10); // y2
+                    }}
+                    document.getElementById('canvas-boxes-data').value = JSON.stringify(boxes);
+                    redraw();
+                }} else if (creatingBox && createStart) {{
+                    dragStart = {{x: x, y: y}};
+                    redraw();
+                }} else {{
+                    // Update cursor
+                    const corner = getCornerAt(x, y);
+                    if (corner) {{
+                        canvas.style.cursor = 'move';
+                    }} else {{
+                        canvas.style.cursor = 'crosshair';
+                    }}
+                }}
+            }});
+            
+            canvas.addEventListener('mouseup', function(e) {{
+                if (creatingBox && createStart && dragStart) {{
+                    const x1 = Math.min(createStart.x, dragStart.x);
+                    const y1 = Math.min(createStart.y, dragStart.y);
+                    const x2 = Math.max(createStart.x, dragStart.x);
+                    const y2 = Math.max(createStart.y, dragStart.y);
+                    
+                    // Only add if box has minimum size
+                    if (x2 - x1 > 10 && y2 - y1 > 10) {{
+                        boxes.push({{
+                            bbox: [x1, y1, x2, y2],
+                            label: 'knot',
+                            confidence: 1.0,
+                            source: 'manual'
+                        }});
+                        document.getElementById('canvas-boxes-data').value = JSON.stringify(boxes);
+                        redraw();
+                        
+                        // Trigger save (this will be handled by the parent)
+                        if (window.saveAnnotations) {{
+                            window.saveAnnotations(boxes);
+                        }}
+                    }}
+                }}
+                
+                isDragging = false;
+                creatingBox = false;
+                selectedCorner = null;
+                selectedBoxIndex = -1;
+                createStart = null;
+                dragStart = null;
+                canvas.style.cursor = 'crosshair';
+            }});
+            
+            // Make boxes available globally for saving
+            window.getCurrentBoxes = function() {{
+                return JSON.stringify(boxes);
+            }};
+            
+            // Function to update boxes from external call
+            window.updateBoxes = function(newBoxes) {{
+                boxes = newBoxes;
+                redraw();
+            }};
+            
+        }})();
+        </script>
+        """
+        
+        return html
     
     def _format_boxes_text(self, boxes: list[dict]) -> str:
         """Format boxes for display."""
@@ -467,7 +679,7 @@ class AnnotationApp:
         
         return "\n".join(lines)
     
-    def load_image(self, direction: str = "current") -> tuple[Image.Image, str, str]:
+    def load_image(self, direction: str = "current") -> tuple[str, str, str]:
         """Load image (current/next/prev)."""
         if direction == "next":
             self.current_idx = min(self.current_idx + 1, len(self.image_paths) - 1)
@@ -476,21 +688,21 @@ class AnnotationApp:
         
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No images</div>", "", "No images"
         
         # Load existing annotations
         boxes = self.annotations.get(filename, [])
-        img_with_boxes = self.draw_boxes_on_image(img, boxes) if boxes else img
+        img_html = self.generate_interactive_canvas(boxes)
         boxes_text = self._format_boxes_text(boxes)
         info = f"Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
         
-        return img_with_boxes, boxes_text, info
+        return img_html, boxes_text, info
     
-    def add_box_manual(self, x1: int, y1: int, x2: int, y2: int) -> tuple[Image.Image, str, str]:
+    def add_box_manual(self, x1: int, y1: int, x2: int, y2: int) -> tuple[str, str, str]:
         """Manually add a bounding box."""
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No images</div>", "", "No images"
         
         # Add box
         box = {
@@ -507,17 +719,17 @@ class AnnotationApp:
         
         # Redraw
         boxes = self.annotations[filename]
-        img_with_boxes = self.draw_boxes_on_image(img, boxes)
+        img_html = self.generate_interactive_canvas(boxes)
         boxes_text = self._format_boxes_text(boxes)
         info = f"✓ Added box: {len(boxes)} total | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
         
-        return img_with_boxes, boxes_text, info
+        return img_html, boxes_text, info
     
-    def delete_last_box(self) -> tuple[Image.Image, str, str]:
+    def delete_last_box(self) -> tuple[str, str, str]:
         """Delete the last box from current image."""
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No images</div>", "", "No images"
         
         if filename in self.annotations and self.annotations[filename]:
             self.annotations[filename].pop()
@@ -525,34 +737,36 @@ class AnnotationApp:
         
         # Redraw
         boxes = self.annotations.get(filename, [])
-        img_with_boxes = self.draw_boxes_on_image(img, boxes) if boxes else img
+        img_html = self.generate_interactive_canvas(boxes)
         boxes_text = self._format_boxes_text(boxes)
         info = f"✓ Deleted last box: {len(boxes)} remaining | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
         
-        return img_with_boxes, boxes_text, info
+        return img_html, boxes_text, info
     
-    def clear_boxes(self) -> tuple[Image.Image, str, str]:
+    def clear_boxes(self) -> tuple[str, str, str]:
         """Clear all boxes from current image."""
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No images</div>", "", "No images"
         
         self.annotations[filename] = []
         self._save_annotations()
         
+        boxes = []
+        img_html = self.generate_interactive_canvas(boxes)
         boxes_text = "No annotations"
         info = f"✓ Cleared all boxes | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
         
-        return img, boxes_text, info
+        return img_html, boxes_text, info
     
-    def auto_label_current(self, threshold: float = 0.5) -> tuple[Image.Image, str, str]:
+    def auto_label_current(self, threshold: float = 0.5) -> tuple[str, str, str]:
         """Auto-label current image using loaded model."""
         if not self.model:
-            return None, "", "❌ No model loaded"
+            return "<div>No model loaded</div>", "", "❌ No model loaded"
         
         img, filename = self.get_current_image()
         if not img:
-            return None, "", "No images"
+            return "<div>No images</div>", "", "No images"
         
         try:
             # Run inference based on model type
@@ -598,14 +812,41 @@ class AnnotationApp:
             self._save_annotations()
             
             # Redraw
-            img_with_boxes = self.draw_boxes_on_image(img, self.annotations[filename])
+            img_html = self.generate_interactive_canvas(self.annotations[filename])
             boxes_text = self._format_boxes_text(self.annotations[filename])
             info = f"🤖 Auto-labeled: {len(boxes)} detections | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
             
-            return img_with_boxes, boxes_text, info
+            return img_html, boxes_text, info
             
         except Exception as e:
-            return img, self._format_boxes_text(self.annotations.get(filename, [])), f"❌ Auto-label failed: {e}"
+            boxes = self.annotations.get(filename, [])
+            img_html = self.generate_interactive_canvas(boxes)
+            return img_html, self._format_boxes_text(boxes), f"❌ Auto-label failed: {e}"
+    
+    def save_canvas_changes(self, boxes_json: str) -> tuple[str, str, str]:
+        """Save changes made in the interactive canvas."""
+        img, filename = self.get_current_image()
+        if not img:
+            return "<div>No images</div>", "", "No images"
+        
+        try:
+            # Parse the boxes from JSON
+            if boxes_json:
+                boxes = json.loads(boxes_json)
+                self.annotations[filename] = boxes
+                self._save_annotations()
+                info = f"✓ Saved {len(boxes)} boxes | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
+            else:
+                boxes = self.annotations.get(filename, [])
+                info = f"✓ No changes to save | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
+        except json.JSONDecodeError:
+            boxes = self.annotations.get(filename, [])
+            info = f"❌ Invalid boxes data | Image {self.current_idx + 1}/{len(self.image_paths)}: {filename}"
+        
+        img_html = self.generate_interactive_canvas(boxes)
+        boxes_text = self._format_boxes_text(boxes)
+        
+        return img_html, boxes_text, info
     
     def _save_annotations(self):
         """Save annotations to JSON file."""
@@ -1041,12 +1282,16 @@ def create_ui(app: AnnotationApp) -> gr.Blocks:
         
         with gr.Row():
             with gr.Column(scale=3):
-                image_display = gr.Image(label="Current Image", type="pil")
+                image_display = gr.HTML(label="Current Image", value="<div style='width: 800px; height: 400px; border: 1px solid #ccc; display: flex; align-items: center; justify-content: center; color: #666;'>Load images from Settings to start annotating</div>")
                 
                 with gr.Row():
                     prev_btn = gr.Button("⬅️ Previous")
                     next_btn = gr.Button("Next ➡️")
                     auto_label_btn = gr.Button("🤖 Auto-Label", variant="primary")
+                    save_canvas_btn = gr.Button("💾 Save Canvas Changes")
+                
+                # Hidden textbox to store canvas boxes data
+                canvas_boxes_data = gr.Textbox(visible=False)
                 
                 with gr.Row():
                     threshold_slider = gr.Slider(0.1, 0.9, DEFAULT_DETECTION_THRESHOLD, label="Detection Threshold")
@@ -1276,6 +1521,19 @@ def create_ui(app: AnnotationApp) -> gr.Blocks:
             outputs=[image_display, boxes_text, info_text]
         )
         
+        save_canvas_btn.click(
+            app.save_canvas_changes,
+            inputs=[canvas_boxes_data],
+            outputs=[image_display, boxes_text, info_text],
+            js="""() => {
+                const hiddenInput = document.getElementById('canvas-boxes-data');
+                if (hiddenInput) {
+                    return hiddenInput.value;
+                }
+                return '';
+            }"""
+        )
+        
         add_box_btn.click(
             app.add_box_manual,
             inputs=[x1_input, y1_input, x2_input, y2_input],