diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
index 51cab6f14b..ac5ad8ea60 100644
--- a/libavformat/rtpenc.c
+++ b/libavformat/rtpenc.c
@@ -226,7 +226,6 @@ static int rtp_write_header(AVFormatContext *s1)
         if (!s->max_frames_per_packet)
             s->max_frames_per_packet = 15;
         s->max_frames_per_packet = av_clip(s->max_frames_per_packet, 1, 15);
-        s->max_payload_size -= 6; // ident+frag+tdt/vdt+pkt_num+pkt_length
         s->num_frames = 0;
         goto defaultcase;
     case AV_CODEC_ID_ADPCM_G722:
diff --git a/libavformat/rtpenc_xiph.c b/libavformat/rtpenc_xiph.c
index d5936774ba..0564b06531 100644
--- a/libavformat/rtpenc_xiph.c
+++ b/libavformat/rtpenc_xiph.c
@@ -36,7 +36,7 @@ void ff_rtp_send_xiph(AVFormatContext *s1, const uint8_t *buff, int size)
     int max_pkt_size, xdt, frag;
     uint8_t *q;
 
-    max_pkt_size = s->max_payload_size;
+    max_pkt_size = s->max_payload_size - 6; // ident+frag+tdt/vdt+pkt_num+pkt_length
 
     // set xiph data type
     switch (*buff) {