From: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
To: linux-media@vger.kernel.org
Cc: Dave Stevenson <dave.stevenson@raspberrypi.com>,
David Plowman <david.plowman@raspberrypi.com>,
Jean-Michel Hautbois <jeanmichel.hautbois@yoseli.org>,
Hans Verkuil <hverkuil-cisco@xs4all.nl>,
Naushir Patuck <naush@raspberrypi.com>,
Sakari Ailus <sakari.ailus@linux.intel.com>,
kernel-list@raspberrypi.com,
linux-rpi-kernel@lists.infradead.org,
Florian Fainelli <florian.fainelli@broadcom.com>,
Ray Jui <rjui@broadcom.com>,
Scott Branden <sbranden@broadcom.com>,
bcm-kernel-feedback-list@broadcom.com
Subject: [PATCH v7 05/15] media: i2c: imx219: Add embedded data support
Date: Mon, 25 Mar 2024 00:08:41 +0200 [thread overview]
Message-ID: <20240324220854.15010-6-laurent.pinchart@ideasonboard.com> (raw)
In-Reply-To: <20240324220854.15010-1-laurent.pinchart@ideasonboard.com>
The IMX219 generates embedded data unconditionally. Report it as an
additional stream, with a new internal embedded data pad, and update
subdev operations accordingly.
Signed-off-by: Laurent Pinchart <laurent.pinchart@ideasonboard.com>
---
Changes since v6:
- Get format from IMX219_STREAM_IMAGE in imx219_set_ctrl()
- Fix mbus code for second stream in imx219_get_frame_desc()
- Set V4L2_SUBDEV_ROUTE_FL_IMMUTABLE flag on route
---
drivers/media/i2c/imx219.c | 188 +++++++++++++++++++++++++++++++------
1 file changed, 160 insertions(+), 28 deletions(-)
diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c
index fa64bc402c9a..86a0ebf6d65f 100644
--- a/drivers/media/i2c/imx219.c
+++ b/drivers/media/i2c/imx219.c
@@ -149,6 +149,9 @@
#define IMX219_PIXEL_ARRAY_WIDTH 3280U
#define IMX219_PIXEL_ARRAY_HEIGHT 2464U
+/* Embedded metadata stream height */
+#define IMX219_EMBEDDED_DATA_HEIGHT 2U
+
/* Mode : resolution and related config&values */
struct imx219_mode {
/* Frame width */
@@ -317,9 +320,15 @@ static const struct imx219_mode supported_modes[] = {
enum imx219_pad_ids {
IMX219_PAD_SOURCE,
IMX219_PAD_IMAGE,
+ IMX219_PAD_EDATA,
IMX219_NUM_PADS,
};
+enum imx219_stream_ids {
+ IMX219_STREAM_IMAGE,
+ IMX219_STREAM_EDATA,
+};
+
struct imx219 {
struct v4l2_subdev sd;
struct media_pad pads[IMX219_NUM_PADS];
@@ -382,7 +391,8 @@ static int imx219_set_ctrl(struct v4l2_ctrl *ctrl)
int ret = 0;
state = v4l2_subdev_get_locked_active_state(&imx219->sd);
- format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE);
+ format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_IMAGE);
if (ctrl->id == V4L2_CID_VBLANK) {
int exposure_max, exposure_def;
@@ -610,6 +620,25 @@ static unsigned int imx219_format_bpp(u32 code)
}
}
+/* Return the embedded data format corresponding to an image format. */
+static u32 imx219_format_edata(u32 code)
+{
+ switch (code) {
+ case MEDIA_BUS_FMT_SRGGB8_1X8:
+ case MEDIA_BUS_FMT_SGRBG8_1X8:
+ case MEDIA_BUS_FMT_SGBRG8_1X8:
+ case MEDIA_BUS_FMT_SBGGR8_1X8:
+ return MEDIA_BUS_FMT_META_8;
+
+ case MEDIA_BUS_FMT_SRGGB10_1X10:
+ case MEDIA_BUS_FMT_SGRBG10_1X10:
+ case MEDIA_BUS_FMT_SGBRG10_1X10:
+ case MEDIA_BUS_FMT_SBGGR10_1X10:
+ default:
+ return MEDIA_BUS_FMT_META_10;
+ }
+}
+
static int imx219_set_framefmt(struct imx219 *imx219,
struct v4l2_subdev_state *state)
{
@@ -619,7 +648,8 @@ static int imx219_set_framefmt(struct imx219 *imx219,
u64 bin_h, bin_v;
int ret = 0;
- format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE);
+ format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_IMAGE);
crop = v4l2_subdev_state_get_crop(state, IMX219_PAD_IMAGE);
bpp = imx219_format_bpp(format->code);
@@ -774,17 +804,33 @@ static int imx219_enum_mbus_code(struct v4l2_subdev *sd,
{
struct imx219 *imx219 = to_imx219(sd);
- if (code->pad == IMX219_PAD_IMAGE) {
+ switch (code->pad) {
+ case IMX219_PAD_IMAGE:
/* The internal image pad is hardwired to the native format. */
- if (code->index)
+ if (code->index > 0)
return -EINVAL;
code->code = IMX219_NATIVE_FORMAT;
- } else {
- /*
- * On the source pad, the sensor supports multiple raw formats
- * with different bit depths.
- */
+ return 0;
+
+ case IMX219_PAD_EDATA:
+ if (code->index > 0)
+ return -EINVAL;
+
+ code->code = MEDIA_BUS_FMT_CCS_EMBEDDED;
+ return 0;
+
+ case IMX219_PAD_SOURCE:
+ default:
+ break;
+ }
+
+ /*
+ * On the source pad, the sensor supports multiple image raw formats
+ * with different bit depths. The embedded data format bit depth
+ * follows the image stream.
+ */
+ if (code->stream == IMX219_STREAM_IMAGE) {
u32 format;
if (code->index >= (ARRAY_SIZE(imx219_mbus_formats) / 4))
@@ -792,6 +838,15 @@ static int imx219_enum_mbus_code(struct v4l2_subdev *sd,
format = imx219_mbus_formats[code->index * 4];
code->code = imx219_get_format_code(imx219, format);
+ } else {
+ struct v4l2_mbus_framefmt *fmt;
+
+ if (code->index > 0)
+ return -EINVAL;
+
+ fmt = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_EDATA);
+ code->code = fmt->code;
}
return 0;
@@ -803,7 +858,8 @@ static int imx219_enum_frame_size(struct v4l2_subdev *sd,
{
struct imx219 *imx219 = to_imx219(sd);
- if (fse->pad == IMX219_PAD_IMAGE) {
+ switch (fse->pad) {
+ case IMX219_PAD_IMAGE:
if (fse->code != IMX219_NATIVE_FORMAT || fse->index > 0)
return -EINVAL;
@@ -811,7 +867,24 @@ static int imx219_enum_frame_size(struct v4l2_subdev *sd,
fse->max_width = IMX219_NATIVE_WIDTH;
fse->min_height = IMX219_NATIVE_HEIGHT;
fse->max_height = IMX219_NATIVE_HEIGHT;
- } else {
+ return 0;
+
+ case IMX219_PAD_EDATA:
+ if (fse->code != MEDIA_BUS_FMT_CCS_EMBEDDED || fse->index > 0)
+ return -EINVAL;
+
+ fse->min_width = IMX219_NATIVE_WIDTH;
+ fse->max_width = IMX219_NATIVE_WIDTH;
+ fse->min_height = IMX219_EMBEDDED_DATA_HEIGHT;
+ fse->max_height = IMX219_EMBEDDED_DATA_HEIGHT;
+ return 0;
+
+ case IMX219_PAD_SOURCE:
+ default:
+ break;
+ }
+
+ if (fse->stream == IMX219_STREAM_IMAGE) {
if (fse->code != imx219_get_format_code(imx219, fse->code) ||
fse->index >= ARRAY_SIZE(supported_modes))
return -EINVAL;
@@ -820,6 +893,21 @@ static int imx219_enum_frame_size(struct v4l2_subdev *sd,
fse->max_width = fse->min_width;
fse->min_height = supported_modes[fse->index].height;
fse->max_height = fse->min_height;
+ } else {
+ struct v4l2_mbus_framefmt *fmt;
+
+ fmt = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_EDATA);
+ if (fse->code != fmt->code)
+ return -EINVAL;
+
+ if (fse->index)
+ return -EINVAL;
+
+ fse->min_width = fmt->width;
+ fse->max_width = fmt->width;
+ fse->min_height = IMX219_EMBEDDED_DATA_HEIGHT;
+ fse->max_height = IMX219_EMBEDDED_DATA_HEIGHT;
}
return 0;
@@ -831,6 +919,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
{
struct imx219 *imx219 = to_imx219(sd);
const struct imx219_mode *mode;
+ struct v4l2_mbus_framefmt *ed_format;
struct v4l2_mbus_framefmt *format;
struct v4l2_rect *compose;
struct v4l2_rect *crop;
@@ -838,9 +927,9 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
/*
* The driver is mode-based, the format can be set on the source pad
- * only.
+ * only, and only for the image streeam.
*/
- if (fmt->pad != IMX219_PAD_SOURCE)
+ if (fmt->pad != IMX219_PAD_SOURCE || fmt->stream != IMX219_STREAM_IMAGE)
return v4l2_subdev_get_fmt(sd, state, fmt);
/*
@@ -897,15 +986,31 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd,
* No mode use digital crop, the source pad crop rectangle size and
* format are thus identical to the image pad compose rectangle.
*/
- crop = v4l2_subdev_state_get_crop(state, IMX219_PAD_SOURCE);
+ crop = v4l2_subdev_state_get_crop(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_IMAGE);
crop->left = 0;
crop->top = 0;
crop->width = fmt->format.width;
crop->height = fmt->format.height;
- format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE);
+ format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_IMAGE);
*format = fmt->format;
+ /*
+ * Finally, update the formats on the sink and source sides of the
+ * embedded data stream.
+ */
+ ed_format = v4l2_subdev_state_get_format(state, IMX219_PAD_EDATA);
+ ed_format->code = imx219_format_edata(format->code);
+ ed_format->width = format->width;
+ ed_format->height = IMX219_EMBEDDED_DATA_HEIGHT;
+ ed_format->field = V4L2_FIELD_NONE;
+
+ format = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_EDATA);
+ *format = *ed_format;
+
if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) {
int exposure_max;
int exposure_def;
@@ -944,6 +1049,13 @@ static int imx219_get_selection(struct v4l2_subdev *sd,
{
struct v4l2_rect *compose;
+ /*
+ * The embedded data stream doesn't support selection rectangles,
+ * neither on the embedded data pad nor on the source pad.
+ */
+ if (sel->pad == IMX219_PAD_EDATA || sel->stream != 0)
+ return -EINVAL;
+
switch (sel->target) {
case V4L2_SEL_TGT_NATIVE_SIZE:
if (sel->pad != IMX219_PAD_IMAGE)
@@ -996,12 +1108,19 @@ static int imx219_get_selection(struct v4l2_subdev *sd,
static int imx219_init_state(struct v4l2_subdev *sd,
struct v4l2_subdev_state *state)
{
- struct v4l2_subdev_route routes[1] = {
+ struct v4l2_subdev_route routes[2] = {
{
.sink_pad = IMX219_PAD_IMAGE,
.sink_stream = 0,
.source_pad = IMX219_PAD_SOURCE,
- .source_stream = 0,
+ .source_stream = IMX219_STREAM_IMAGE,
+ .flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE |
+ V4L2_SUBDEV_ROUTE_FL_IMMUTABLE,
+ }, {
+ .sink_pad = IMX219_PAD_EDATA,
+ .sink_stream = 0,
+ .source_pad = IMX219_PAD_SOURCE,
+ .source_stream = IMX219_STREAM_EDATA,
.flags = V4L2_SUBDEV_ROUTE_FL_ACTIVE |
V4L2_SUBDEV_ROUTE_FL_IMMUTABLE,
},
@@ -1014,7 +1133,7 @@ static int imx219_init_state(struct v4l2_subdev *sd,
struct v4l2_subdev_format fmt = {
.which = V4L2_SUBDEV_FORMAT_TRY,
.pad = IMX219_PAD_SOURCE,
- .stream = 0,
+ .stream = IMX219_STREAM_IMAGE,
.format = {
.code = MEDIA_BUS_FMT_SRGGB10_1X10,
.width = supported_modes[0].width,
@@ -1027,6 +1146,10 @@ static int imx219_init_state(struct v4l2_subdev *sd,
if (ret)
return ret;
+ /*
+ * Set the image stream format on the source pad. This will be
+ * propagated to all formats and selection rectangles internally.
+ */
imx219_set_pad_format(sd, state, &fmt);
return 0;
@@ -1035,29 +1158,36 @@ static int imx219_init_state(struct v4l2_subdev *sd,
static int imx219_get_frame_desc(struct v4l2_subdev *sd, unsigned int pad,
struct v4l2_mbus_frame_desc *fd)
{
- const struct v4l2_mbus_framefmt *fmt;
struct v4l2_subdev_state *state;
- u32 code;
+ u32 img_code;
+ u32 ed_code;
if (pad != IMX219_PAD_SOURCE)
return -EINVAL;
state = v4l2_subdev_lock_and_get_active_state(sd);
- fmt = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE, 0);
- code = fmt->code;
+ img_code = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_IMAGE)->code;
+ ed_code = v4l2_subdev_state_get_format(state, IMX219_PAD_SOURCE,
+ IMX219_STREAM_EDATA)->code;
v4l2_subdev_unlock_state(state);
fd->type = V4L2_MBUS_FRAME_DESC_TYPE_CSI2;
- fd->num_entries = 1;
+ fd->num_entries = 2;
memset(fd->entry, 0, sizeof(fd->entry));
- fd->entry[0].pixelcode = code;
- fd->entry[0].stream = 0;
+ fd->entry[0].pixelcode = img_code;
+ fd->entry[0].stream = IMX219_STREAM_IMAGE;
fd->entry[0].bus.csi2.vc = 0;
- fd->entry[0].bus.csi2.dt = imx219_format_bpp(code) == 8
+ fd->entry[0].bus.csi2.dt = imx219_format_bpp(img_code) == 8
? MIPI_CSI2_DT_RAW8 : MIPI_CSI2_DT_RAW10;
+ fd->entry[1].pixelcode = ed_code;
+ fd->entry[1].stream = IMX219_STREAM_EDATA;
+ fd->entry[1].bus.csi2.vc = 0;
+ fd->entry[1].bus.csi2.dt = MIPI_CSI2_DT_EMBEDDED_8B;
+
return 0;
}
@@ -1321,12 +1451,14 @@ static int imx219_probe(struct i2c_client *client)
/*
* Initialize the pads. To preserve backward compatibility with
* userspace that used the sensor before the introduction of the
- * internal image pad, the external source pad is numbered 0 and the
- * internal image pad numbered 1.
+ * internal pads, the external source pad is numbered 0 and the internal
+ * image and embedded data pads numbered 1 and 2 respectively.
*/
imx219->pads[IMX219_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE;
imx219->pads[IMX219_PAD_IMAGE].flags = MEDIA_PAD_FL_SINK
| MEDIA_PAD_FL_INTERNAL;
+ imx219->pads[IMX219_PAD_EDATA].flags = MEDIA_PAD_FL_SINK
+ | MEDIA_PAD_FL_INTERNAL;
ret = media_entity_pads_init(&imx219->sd.entity,
ARRAY_SIZE(imx219->pads), imx219->pads);
--
Regards,
Laurent Pinchart
next prev parent reply other threads:[~2024-03-24 22:09 UTC|newest]
Thread overview: 40+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-03-24 22:08 [PATCH v7 00/15] media: Add driver for the Raspberry Pi <5 CSI-2 receiver Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 01/15] media: i2c: imx219: Inline imx219_update_pad_format() in its caller Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 02/15] media: i2c: imx219: Add internal image sink pad Laurent Pinchart
2024-03-27 9:51 ` Tomi Valkeinen
2024-03-28 16:09 ` Laurent Pinchart
2024-03-28 16:18 ` Tomi Valkeinen
2024-03-28 16:56 ` Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 03/15] media: i2c: imx219: Report internal routes to userspace Laurent Pinchart
2024-03-27 9:56 ` Tomi Valkeinen
2024-04-04 8:19 ` Tomi Valkeinen
2024-04-04 8:29 ` Sakari Ailus
2024-04-04 10:29 ` Tomi Valkeinen
2024-04-04 10:34 ` Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 04/15] media: i2c: imx219: Report streams using frame descriptors Laurent Pinchart
2024-03-27 10:08 ` Tomi Valkeinen
2024-03-28 17:17 ` Laurent Pinchart
2024-03-24 22:08 ` Laurent Pinchart [this message]
2024-03-26 23:30 ` [PATCH v7 05/15] media: i2c: imx219: Add embedded data support Sakari Ailus
2024-03-27 0:22 ` Laurent Pinchart
2024-03-27 6:52 ` Sakari Ailus
2024-03-27 10:51 ` Tomi Valkeinen
2024-03-28 21:49 ` Laurent Pinchart
2024-03-29 9:13 ` Tomi Valkeinen
2024-03-24 22:08 ` [PATCH v7 06/15] media: v4l: Add V4L2-PIX-FMT-Y12P format Laurent Pinchart
2024-03-27 11:08 ` Tomi Valkeinen
2024-03-27 11:18 ` Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 07/15] media: v4l: Add V4L2-PIX-FMT-Y14P format Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 08/15] dt-bindings: media: Add bindings for bcm2835-unicam Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 09/15] media: bcm2835-unicam: Add support for CCP2/CSI2 camera interface Laurent Pinchart
2024-03-27 11:21 ` Tomi Valkeinen
2024-04-01 13:52 ` Laurent Pinchart
2024-04-02 6:00 ` Tomi Valkeinen
2024-04-02 6:05 ` Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 10/15] ARM: dts: bcm2835-rpi: Move firmware-clocks from bcm2711 to bcm2835 Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 11/15] ARM: dts: bcm2835: Add Unicam CSI nodes Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 12/15] ARM: dts: bcm2711-rpi: Add pinctrl-based multiplexing for I2C0 Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 13/15] ARM: dts: bcm2711-rpi-cm4-io: Add RTC on I2C0 Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 14/15] ARM: dts: bcm2711-rpi-4-b: Add CAM1 regulator Laurent Pinchart
2024-03-24 22:08 ` [PATCH v7 15/15] [DNI] arm64: dts: broadcom: Add overlay for Raspberry Pi 4B IMX219 camera Laurent Pinchart
2024-03-25 13:55 ` [PATCH v7 00/15] media: Add driver for the Raspberry Pi <5 CSI-2 receiver Rob Herring
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20240324220854.15010-6-laurent.pinchart@ideasonboard.com \
--to=laurent.pinchart@ideasonboard.com \
--cc=bcm-kernel-feedback-list@broadcom.com \
--cc=dave.stevenson@raspberrypi.com \
--cc=david.plowman@raspberrypi.com \
--cc=florian.fainelli@broadcom.com \
--cc=hverkuil-cisco@xs4all.nl \
--cc=jeanmichel.hautbois@yoseli.org \
--cc=kernel-list@raspberrypi.com \
--cc=linux-media@vger.kernel.org \
--cc=linux-rpi-kernel@lists.infradead.org \
--cc=naush@raspberrypi.com \
--cc=rjui@broadcom.com \
--cc=sakari.ailus@linux.intel.com \
--cc=sbranden@broadcom.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.