Chiebot-Mirror
/
FFmpeg
mirror of https://github.com/FFmpeg/FFmpeg.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

http: honor response headers in redirect caching

Browse Source 
add a dictionary that maps "src_url" -> "expiry;dst_url", the dictionary
is checked before issuing an http request, and updated after getting a
3xx redirect response.

the cache expiry is determined according to the following (in desc
priority) -
1. Expires header
2. Cache-Control containing no-cache/no-store (disables caching)
3. Cache-Control s-maxage/max-age
4. Http codes 301/308 are cached indefinitely, other codes are not
   cached
release/5.1
erankor 3 years ago committed by Ronald S. Bultje
parent 641c4346b3
commit 625ea2d2a9
1 changed files with 148 additions and 38 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 186
      libavformat/http.c

186
libavformat/http.c
Unescape View File

@ -48,6 +48,7 @@
* path names). */
#define BUFFER_SIZE (MAX_URL_SIZE + HTTP_HEADERS_SIZE)
#define MAX_REDIRECTS 8
#define MAX_CACHED_REDIRECTS 32
#define HTTP_SINGLE 1
#define HTTP_MUTLI 2
#define MAX_EXPIRY 19
@ -129,6 +130,9 @@ typedef struct HTTPContext {
HandshakeState handshake_step;
int is_connected_server;
int short_seek_size;
int64_t expires;
char *new_location;
AVDictionary *redirect_cache;
} HTTPContext;
#define OFFSET(x) offsetof(HTTPContext, x)
@ -177,8 +181,8 @@ static const AVOption options[] = {
static int http_connect(URLContext *h, const char *path, const char *local_path,
const char *hoststr, const char *auth,
const char *proxyauth, int *new_location);
static int http_read_header(URLContext *h, int *new_location);
const char *proxyauth);
static int http_read_header(URLContext *h);
static int http_shutdown(URLContext *h, int flags);
void ff_http_init_auth_state(URLContext *dest, const URLContext *src)
@ -199,7 +203,7 @@ static int http_open_cnx_internal(URLContext *h, AVDictionary **options)
char auth[1024], proxyauth[1024] = "";
char path1[MAX_URL_SIZE], sanitized_path[MAX_URL_SIZE + 1];
char buf[1024], urlbuf[MAX_URL_SIZE];
int port, use_proxy, err, location_changed = 0;
int port, use_proxy, err;
HTTPContext *s = h->priv_data;
av_url_split(proto, sizeof(proto), auth, sizeof(auth),
@ -259,12 +263,8 @@ static int http_open_cnx_internal(URLContext *h, AVDictionary **options)
return err;
}
err = http_connect(h, path, local_path, hoststr,
auth, proxyauth, &location_changed);
if (err < 0)
return err;
return location_changed;
return http_connect(h, path, local_path, hoststr,
auth, proxyauth);
}
static int http_should_reconnect(HTTPContext *s, int err)
@ -300,31 +300,87 @@ static int http_should_reconnect(HTTPContext *s, int err)
return av_match_list(http_code, s->reconnect_on_http_error, ',') > 0;
}
static char *redirect_cache_get(HTTPContext *s)
{
AVDictionaryEntry *re;
int64_t expiry;
char *delim;
re = av_dict_get(s->redirect_cache, s->location, NULL, AV_DICT_MATCH_CASE);
if (!re) {
return NULL;
}
delim = strchr(re->value, ';');
if (!delim) {
return NULL;
}
expiry = strtoll(re->value, NULL, 10);
if (time(NULL) > expiry) {
return NULL;
}
return delim + 1;
}
static int redirect_cache_set(HTTPContext *s, const char *source, const char *dest, int64_t expiry)
{
char *value;
int ret;
value = av_asprintf("%"PRIi64";%s", expiry, dest);
if (!value) {
return AVERROR(ENOMEM);
}
ret = av_dict_set(&s->redirect_cache, source, value, AV_DICT_MATCH_CASE | AV_DICT_DONT_STRDUP_VAL);
if (ret < 0) {
av_free(value);
return ret;
}
return 0;
}
/* return non zero if error */
static int http_open_cnx(URLContext *h, AVDictionary **options)
{
HTTPAuthType cur_auth_type, cur_proxy_auth_type;
HTTPContext *s = h->priv_data;
int location_changed, attempts = 0, redirects = 0;
int ret, attempts = 0, redirects = 0;
int reconnect_delay = 0;
uint64_t off;
char *cached;
redo:
cached = redirect_cache_get(s);
if (cached) {
av_free(s->location);
s->location = av_strdup(cached);
if (!s->location) {
ret = AVERROR(ENOMEM);
goto fail;
}
goto redo;
}
av_dict_copy(options, s->chained_options, 0);
cur_auth_type = s->auth_state.auth_type;
cur_proxy_auth_type = s->auth_state.auth_type;
off = s->off;
location_changed = http_open_cnx_internal(h, options);
if (location_changed < 0) {
if (!http_should_reconnect(s, location_changed) ||
ret = http_open_cnx_internal(h, options);
if (ret < 0) {
if (!http_should_reconnect(s, ret) ||
reconnect_delay > s->reconnect_delay_max)
goto fail;
av_log(h, AV_LOG_WARNING, "Will reconnect at %"PRIu64" in %d second(s).\n", off, reconnect_delay);
location_changed = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
if (location_changed != AVERROR(ETIMEDOUT))
ret = ff_network_sleep_interruptible(1000U * 1000 * reconnect_delay, &h->interrupt_callback);
if (ret != AVERROR(ETIMEDOUT))
goto fail;
reconnect_delay = 1 + 2 * reconnect_delay;
@ -354,16 +410,28 @@ redo:
}
if ((s->http_code == 301 || s->http_code == 302 ||
s->http_code == 303 || s->http_code == 307 || s->http_code == 308) &&
location_changed == 1) {
s->new_location) {
/* url moved, get next */
ffurl_closep(&s->hd);
if (redirects++ >= MAX_REDIRECTS)
return AVERROR(EIO);
if (!s->expires) {
s->expires = (s->http_code == 301 || s->http_code == 308) ? INT64_MAX : -1;
}
if (s->expires > time(NULL) && av_dict_count(s->redirect_cache) < MAX_CACHED_REDIRECTS) {
redirect_cache_set(s, s->location, s->new_location, s->expires);
}
av_free(s->location);
s->location = s->new_location;
s->new_location = NULL;
/* Restart the authentication process with the new target, which
* might use a different auth mechanism. */
memset(&s->auth_state, 0, sizeof(s->auth_state));
attempts = 0;
location_changed = 0;
goto redo;
}
return 0;
@ -371,8 +439,8 @@ redo:
fail:
if (s->hd)
ffurl_closep(&s->hd);
if (location_changed < 0)
return location_changed;
if (ret < 0)
return ret;
return ff_http_averror(s->http_code, AVERROR(EIO));
}
int ff_http_get_shutdown_status(URLContext *h)
@ -554,7 +622,7 @@ static void handle_http_errors(URLContext *h, int error)
static int http_handshake(URLContext *c)
{
int ret, err, new_location;
int ret, err;
HTTPContext *ch = c->priv_data;
URLContext *cl = ch->hd;
switch (ch->handshake_step) {
@ -569,7 +637,7 @@ static int http_handshake(URLContext *c)
return 2;
case READ_HEADERS:
av_log(c, AV_LOG_TRACE, "Read headers\n");
if ((err = http_read_header(c, &new_location)) < 0) {
if ((err = http_read_header(c)) < 0) {
handle_http_errors(c, err);
return err;
}
@ -666,6 +734,8 @@ bail_out:
if (ret < 0) {
av_dict_free(&s->chained_options);
av_dict_free(&s->cookie_dict);
av_dict_free(&s->redirect_cache);
av_freep(&s->new_location);
av_freep(&s->uri);
}
return ret;
@ -753,14 +823,13 @@ static int check_http_code(URLContext *h, int http_code, const char *end)
static int parse_location(HTTPContext *s, const char *p)
{
char redirected_location[MAX_URL_SIZE], *new_loc;
char redirected_location[MAX_URL_SIZE];
ff_make_absolute_url(redirected_location, sizeof(redirected_location),
s->location, p);
new_loc = av_strdup(redirected_location);
if (!new_loc)
av_freep(&s->new_location);
s->new_location = av_strdup(redirected_location);
if (!s->new_location)
return AVERROR(ENOMEM);
av_free(s->location);
s->location = new_loc;
return 0;
}
@ -983,8 +1052,43 @@ static int cookie_string(AVDictionary *dict, char **cookies)
return 0;
}
static int process_line(URLContext *h, char *line, int line_count,
int *new_location)
static void parse_expires(HTTPContext *s, const char *p)
{
struct tm tm;
if (!parse_set_cookie_expiry_time(p, &tm)) {
s->expires = av_timegm(&tm);
}
}
static void parse_cache_control(HTTPContext *s, const char *p)
{
char *age;
int offset;
/* give 'Expires' higher priority over 'Cache-Control' */
if (s->expires) {
return;
}
if (av_stristr(p, "no-cache") || av_stristr(p, "no-store")) {
s->expires = -1;
return;
}
age = av_stristr(p, "s-maxage=");
offset = 9;
if (!age) {
age = av_stristr(p, "max-age=");
offset = 8;
}
if (age) {
s->expires = time(NULL) + atoi(p + offset);
}
}
static int process_line(URLContext *h, char *line, int line_count)
{
HTTPContext *s = h->priv_data;
const char *auto_method = h->flags & AVIO_FLAG_READ ? "POST" : "GET";
@ -1081,7 +1185,6 @@ static int process_line(URLContext *h, char *line, int line_count,
if (!av_strcasecmp(tag, "Location")) {
if ((ret = parse_location(s, p)) < 0)
return ret;
*new_location = 1;
} else if (!av_strcasecmp(tag, "Content-Length") &&
s->filesize == UINT64_MAX) {
s->filesize = strtoull(p, NULL, 10);
@ -1124,6 +1227,10 @@ static int process_line(URLContext *h, char *line, int line_count,
} else if (!av_strcasecmp(tag, "Content-Encoding")) {
if ((ret = parse_content_encoding(h, p)) < 0)
return ret;
} else if (!av_strcasecmp(tag, "Expires")) {
parse_expires(s, p);
} else if (!av_strcasecmp(tag, "Cache-Control")) {
parse_cache_control(s, p);
}
}
return 1;
@ -1229,12 +1336,14 @@ static inline int has_header(const char *str, const char *header)
return av_stristart(str, header + 2, NULL) || av_stristr(str, header);
}
static int http_read_header(URLContext *h, int *new_location)
static int http_read_header(URLContext *h)
{
HTTPContext *s = h->priv_data;
char line[MAX_URL_SIZE];
int err = 0;
av_freep(&s->new_location);
s->expires = 0;
s->chunksize = UINT64_MAX;
for (;;) {
@ -1243,7 +1352,7 @@ static int http_read_header(URLContext *h, int *new_location)
av_log(h, AV_LOG_TRACE, "header='%s'\n", line);
err = process_line(h, line, s->line_count, new_location);
err = process_line(h, line, s->line_count);
if (err < 0)
return err;
if (err == 0)
@ -1294,7 +1403,7 @@ static void bprint_escaped_path(AVBPrint *bp, const char *path)
static int http_connect(URLContext *h, const char *path, const char *local_path,
const char *hoststr, const char *auth,
const char *proxyauth, int *new_location)
const char *proxyauth)
{
HTTPContext *s = h->priv_data;
int post, err;
@ -1438,11 +1547,11 @@ static int http_connect(URLContext *h, const char *path, const char *local_path,
}
/* wait for header */
err = http_read_header(h, new_location);
err = http_read_header(h);
if (err < 0)
goto done;
if (*new_location)
if (s->new_location)
s->off = off;
err = (off == s->off) ? 0 : -1;
@ -1564,7 +1673,7 @@ static int64_t http_seek_internal(URLContext *h, int64_t off, int whence, int fo
static int http_read_stream(URLContext *h, uint8_t *buf, int size)
{
HTTPContext *s = h->priv_data;
int err, new_location, read_ret;
int err, read_ret;
int64_t seek_ret;
int reconnect_delay = 0;
@ -1572,7 +1681,7 @@ static int http_read_stream(URLContext *h, uint8_t *buf, int size)
return AVERROR_EOF;
if (s->end_chunked_post && !s->end_header) {
err = http_read_header(h, &new_location);
err = http_read_header(h);
if (err < 0)
return err;
}
@ -1785,6 +1894,8 @@ static int http_close(URLContext *h)
ffurl_closep(&s->hd);
av_dict_free(&s->chained_options);
av_dict_free(&s->cookie_dict);
av_dict_free(&s->redirect_cache);
av_freep(&s->new_location);
av_freep(&s->uri);
return ret;
}
@ -1944,7 +2055,6 @@ static int http_proxy_open(URLContext *h, const char *uri, int flags)
int port, ret = 0, attempts = 0;
HTTPAuthType cur_auth_type;
char *authstr;
int new_loc;
if( s->seekable == 1 )
h->is_streamed = 0;
@ -1998,7 +2108,7 @@ redo:
* since the client starts the conversation there, so there
* is no extra data that we might buffer up here.
*/
ret = http_read_header(h, &new_loc);
ret = http_read_header(h);
if (ret < 0)
goto fail;

Write Preview
Loading…
Cancel
Save