diff --git a/spa/plugins/bluez5/iso-io.c b/spa/plugins/bluez5/iso-io.c
index 02d74c2a7..6a933764f 100644
--- a/spa/plugins/bluez5/iso-io.c
+++ b/spa/plugins/bluez5/iso-io.c
@@ -682,7 +682,6 @@ void spa_bt_iso_io_set_source_buffer(struct spa_bt_iso_io *this, struct spa_bt_d
 	if (buffer) {
 		/* Take over buffer overrun handling */
 		buffer->no_overrun_drop = true;
-		buffer->buffering = false;
 		buffer->avg_period = ISO_BUFFERING_AVG_PERIOD;
 		buffer->rate_diff_max = ISO_BUFFERING_RATE_DIFF_MAX;
 	}
diff --git a/spa/plugins/bluez5/media-source.c b/spa/plugins/bluez5/media-source.c
index 14896d6b3..f8db97072 100644
--- a/spa/plugins/bluez5/media-source.c
+++ b/spa/plugins/bluez5/media-source.c
@@ -173,6 +173,7 @@ struct impl {
 
 	int seqnum;
 	uint32_t plc_packets;
+	bool initial_buffering;
 
 	uint32_t errqueue_count;
 
@@ -555,6 +556,8 @@ static int produce_plc_data(struct impl *this)
 
 	if (!this->codec->produce_plc)
 		return -ENOTSUP;
+	if (this->initial_buffering)
+		return -EINVAL;
 
 	buf = spa_bt_decode_buffer_get_write(&port->buffer, &avail);
 	res = this->codec->produce_plc(this->codec_data, buf, avail);
@@ -1059,6 +1062,8 @@ static int transport_start(struct impl *this)
 		spa_loop_locked(this->data_loop, do_start_sco_iso_io, 0, NULL, 0, this);
 	}
 
+	this->initial_buffering = true;
+
 	this->transport_started = true;
 
 	set_latency(this, true);
@@ -1875,9 +1880,12 @@ static void process_buffering(struct impl *this)
 		spa_list_append(&port->ready, &buffer->link);
 	}
 
-	if (this->transport->iso_io && this->position)
+	if (this->transport->iso_io && this->position && !this->initial_buffering)
 		spa_bt_iso_io_check_rx_sync(this->transport->iso_io, this->position->clock.position);
 
+	if (!port->buffer.buffering)
+		this->initial_buffering = false;
+
 	if (this->update_delay_event) {
 		int32_t target = spa_bt_decode_buffer_get_target_latency(&port->buffer);
 		uint32_t decoder_delay = 0;