diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
index cfa5f348fd..ae1d3e04c6 100644
--- a/libavformat/rtpenc.c
+++ b/libavformat/rtpenc.c
@@ -223,7 +223,6 @@ static int rtp_write_header(AVFormatContext *s1)
         if (!s->max_frames_per_packet)
             s->max_frames_per_packet = 15;
         s->max_frames_per_packet = av_clip(s->max_frames_per_packet, 1, 15);
-        s->max_payload_size -= 6; // ident+frag+tdt/vdt+pkt_num+pkt_length
         s->num_frames = 0;
         goto defaultcase;
     case AV_CODEC_ID_ADPCM_G722:
diff --git a/libavformat/rtpenc_xiph.c b/libavformat/rtpenc_xiph.c
index ef31c048de..7d8883a353 100644
--- a/libavformat/rtpenc_xiph.c
+++ b/libavformat/rtpenc_xiph.c
@@ -35,7 +35,7 @@ void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
     int max_pkt_size, xdt, frag;
     uint8_t *q;
 
-    max_pkt_size = s->max_payload_size;
+    max_pkt_size = s->max_payload_size - 6; // ident+frag+tdt/vdt+pkt_num+pkt_length
 
     // set xiph data type
     switch (*buff) {