PDFBOX-6077: Add regression tests for stencil-mask pattern fills

pdfbox/pom.xml

@@ -996,6 +996,32 @@
                             <sha512>2ee83beb6658792c6b963010462515ff1187aafe64930c1a7465dc26e86ab5236dc98cb006986e5fcd60876bedef7f00e4e1cb86bbf590bba000a00187af0ee3</sha512>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>PDFBOX-6077-1</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>wget</goal>
+                        </goals>
+                        <configuration>
+                            <url>https://issues.apache.org/jira/secure/attachment/13078553/example.pdf</url>
+                            <outputDirectory>${project.build.directory}/pdfs</outputDirectory>
+                            <outputFileName>PDFBOX-6077-example.pdf</outputFileName>
+                            <sha512>f4faaa68062073ffdfd48119563f1e1a6eff6d9a063188dc2ec59ccc747277fcaa6139936cebf2f3f7b1b99b80ca13934fb4568658a169e0bf25bf4e748de101</sha512>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>PDFBOX-6077-2</id>
+                        <phase>generate-test-resources</phase>
+                        <goals>
+                            <goal>wget</goal>
+                        </goals>
+                        <configuration>
+                            <url>https://issues.apache.org/jira/secure/attachment/13078557/PDFBOX-5842-reduced.pdf</url>
+                            <outputDirectory>${project.build.directory}/pdfs</outputDirectory>
+                            <outputFileName>PDFBOX-5842-reduced.pdf</outputFileName>
+                            <sha512>c66b13230d343d6b0a2de81680f5d1ae7f8d92713acbc453ff36fe085831fa89127e0428d297e7494b87fc932ca8a721744db1fe79a41cc5c3cf7d631fc209fc</sha512>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
         </plugins>

pdfbox/src/main/java/org/apache/pdfbox/rendering/PageDrawer.java

@@ -1135,18 +1135,32 @@ public void drawImage(PDImage pdImage) throws IOException
 
                 // draw the paint
                 Paint paint = getNonStrokingPaint();
+                // PDFBOX-6077: a soft mask's Paint/PaintContext machinery (see
+                // applySoftMaskToPaint()) assumes it is asked to render directly onto the real
+                // page raster, using its own cached absolute page-device coordinates. This
+                // stencil-mask-with-pattern case instead renders into an isolated scratch image
+                // (see the note above about "device scale is not used"), so unwrap any soft mask
+                // here, fill with the plain underlying paint below, and apply the soft mask's own
+                // alpha afterwards by directly looking up its backing raster (applySoftMaskAlpha).
+                SoftMask softMask = paint instanceof SoftMask ? (SoftMask) paint : null;
+                Paint innerPaint = softMask != null ? softMask.getPaint() : paint;
                 Rectangle2D unitRect = new Rectangle2D.Float(0, 0, 1, 1);
                 Rectangle2D bounds = at.createTransformedShape(unitRect).getBounds2D();
                 int w = (int) Math.ceil(bounds.getWidth());
                 int h = (int) Math.ceil(bounds.getHeight());
                 BufferedImage renderedPaint = new BufferedImage(w, h, BufferedImage.TYPE_INT_ARGB);
                 Graphics2D g = (Graphics2D) renderedPaint.getGraphics();
                 g.translate(-bounds.getMinX(), -bounds.getMinY());
-                g.setPaint(paint);
+                g.setPaint(innerPaint);
                 g.setRenderingHints(graphics.getRenderingHints());
                 g.fill(bounds);
                 g.dispose();
 
+                if (softMask != null)
+                {
+                    applySoftMaskAlpha(renderedPaint, bounds, softMask);
+                }
+
                 // draw the mask
                 BufferedImage mask = pdImage.getImage();
                 AffineTransform imageTransform = new AffineTransform(at);
@@ -1225,7 +1239,11 @@ else if (scaleX != 0 && scaleY != 0)
                     {
                         alphaPixel = alpha.getPixel(x, y, alphaPixel);
                         rasterPixel = raster.getPixel(x, y, rasterPixel);
-                        rasterPixel[3] = alphaPixel[0];
+                        // PDFBOX-6077: combine with the paint's own alpha instead of
+                        // overwriting it, so gaps the paint never drew into (e.g. between
+                        // tiles of a tiling pattern) stay transparent instead of turning
+                        // into opaque black.
+                        rasterPixel[3] = rasterPixel[3] * alphaPixel[0] / 255;
                         raster.setPixel(x, y, rasterPixel);
                     }
                 }
@@ -1267,6 +1285,78 @@ else if (scaleX != 0 && scaleY != 0)
         }
     }
 
+    /**
+     * PDFBOX-6077: applies a soft mask's alpha directly to "image", which was filled with the
+     * soft mask's underlying paint but not yet masked by it. This re-implements
+     * {@link SoftMask}'s own alpha lookup (see its SoftPaintContext.getRaster()) rather than
+     * relying on the Paint/PaintContext machinery, because "image" is a small scratch buffer -
+     * not the real page raster that the soft mask's absolute device coordinates are relative to
+     * - and unlike a plain coordinate offset, transforming each pixel individually stays correct
+     * even though this scratch buffer isn't rendered at the page's actual device scale (see the
+     * "device scale is not used" note where this method is called from).
+     *
+     * @param image the ARGB image to apply the soft mask's alpha to, in place.
+     * @param bounds the device-independent bounds (see "at" in drawImage()) that image's pixel
+     * (0, 0) to (image.getWidth(), image.getHeight()) covers.
+     * @param softMask the soft mask to apply.
+     */
+    private void applySoftMaskAlpha(BufferedImage image, Rectangle2D bounds, SoftMask softMask) throws IOException
+    {
+        AffineTransform deviceTransform = graphics.getTransform();
+        Raster maskRaster = softMask.getMask().getRaster();
+        Rectangle2D bboxDevice = softMask.getBBoxDevice();
+        int backdropColorValue = softMask.getBackdropColorValue();
+        PDFunction transferFunction = softMask.getTransferFunction();
+        Float[] map = transferFunction != null ? new Float[256] : null;
+        float[] input = transferFunction != null ? new float[1] : null;
+
+        WritableRaster raster = image.getRaster();
+        int width = image.getWidth();
+        int height = image.getHeight();
+        Point2D.Double point = new Point2D.Double();
+        int[] gray = new int[1];
+        int[] rasterPixel = null;
+        for (int y = 0; y < height; y++)
+        {
+            for (int x = 0; x < width; x++)
+            {
+                point.setLocation(bounds.getMinX() + x, bounds.getMinY() + y);
+                deviceTransform.transform(point, point);
+                int maskX = (int) Math.floor(point.getX() - bboxDevice.getX());
+                int maskY = (int) Math.floor(point.getY() - bboxDevice.getY());
+
+                int alphaScale;
+                if (maskX >= 0 && maskY >= 0 && maskX < maskRaster.getWidth() && maskY < maskRaster.getHeight())
+                {
+                    maskRaster.getPixel(maskX, maskY, gray);
+                    if (transferFunction != null)
+                    {
+                        Float f = map[gray[0]];
+                        if (f == null)
+                        {
+                            input[0] = gray[0] / 255f;
+                            f = transferFunction.eval(input)[0];
+                            map[gray[0]] = f;
+                        }
+                        alphaScale = Math.round(255 * f);
+                    }
+                    else
+                    {
+                        alphaScale = gray[0];
+                    }
+                }
+                else
+                {
+                    alphaScale = backdropColorValue;
+                }
+
+                rasterPixel = raster.getPixel(x, y, rasterPixel);
+                rasterPixel[3] = rasterPixel[3] * alphaScale / 255;
+                raster.setPixel(x, y, rasterPixel);
+            }
+        }
+    }
+
     /**
      * Calculates the subsampling frequency for a given PDImage based on the current transformation
      * and its calculated transform. Extend this method if you want to use your own strategy.

pdfbox/src/main/java/org/apache/pdfbox/rendering/SoftMask.java

@@ -98,6 +98,35 @@ class SoftMask implements Paint
         }
     }
 
+    // PDFBOX-6077: accessors used by PageDrawer to re-implement this soft mask's alpha lookup
+    // directly (with its own, correctly-scaled device coordinates) when this Paint is rendered
+    // into a scratch image that isn't the real page raster, e.g. for a stencil mask filled with
+    // a pattern.
+    Paint getPaint()
+    {
+        return paint;
+    }
+
+    BufferedImage getMask()
+    {
+        return mask;
+    }
+
+    Rectangle2D getBBoxDevice()
+    {
+        return bboxDevice;
+    }
+
+    int getBackdropColorValue()
+    {
+        return bc;
+    }
+
+    PDFunction getTransferFunction()
+    {
+        return transferFunction;
+    }
+
     @Override
     public PaintContext createContext(ColorModel cm, Rectangle deviceBounds,
                                       Rectangle2D userBounds, AffineTransform xform,

pdfbox/src/test/java/org/apache/pdfbox/rendering/TestQuality.java

@@ -56,4 +56,48 @@ void testPDFBox4831() throws IOException
             ValidateXImage.checkIdent(extractedImage, renderedImage);
         }
     }
+
+    /**
+     * PDFBOX-6077: a stencil mask filled with a pattern must not paint the gaps between the
+     * pattern's own tiles as opaque black. Before the fix, the stencil mask's alpha overwrote
+     * the pattern paint's own alpha instead of being combined with it, so any pixel the pattern
+     * didn't itself draw into turned solid black instead of staying transparent.
+     *
+     * @throws IOException
+     */
+    @Test
+    void testPDFBox6077() throws IOException
+    {
+        File file = new File(TARGET_PDF_DIR, "PDFBOX-6077-example.pdf");
+        try (PDDocument doc = Loader.loadPDF(file))
+        {
+            PDFRenderer renderer = new PDFRenderer(doc);
+            BufferedImage renderedImage = renderer.renderImageWithDPI(0, 100);
+            // a gap between the tiling pattern's own painted tiles, which must stay transparent
+            // (i.e. show the white page background) instead of turning opaque black
+            Assertions.assertEquals(0xFFFFFFFF, renderedImage.getRGB(280, 23));
+        }
+    }
+
+    /**
+     * PDFBOX-6077: a soft mask applied to a pattern that is used as a stencil mask fill must
+     * still be visible. Such a pattern is rendered into a separate scratch image rather than
+     * directly onto the page, and the soft mask's own alpha lookup is keyed to absolute
+     * page-device pixel coordinates, so a naive implementation renders it as fully transparent.
+     *
+     * @throws IOException
+     */
+    @Test
+    void testPDFBox5842() throws IOException
+    {
+        File file = new File(TARGET_PDF_DIR, "PDFBOX-5842-reduced.pdf");
+        try (PDDocument doc = Loader.loadPDF(file))
+        {
+            PDFRenderer renderer = new PDFRenderer(doc);
+            BufferedImage renderedImage = renderer.renderImageWithDPI(0, 100);
+            // a pixel within the soft-masked pattern's map marker icon; if the soft mask's alpha
+            // lookup is broken, this whole region renders as blank white instead
+            Assertions.assertNotEquals(0xFFFFFFFF, renderedImage.getRGB(267, 1329));
+        }
+    }
 }