mirror of
https://github.com/flatpak/flatpak.git
synced 2026-01-26 14:13:26 +00:00
Add flatpak_cache_http_uri: cache downloads based on HTTP headers
Add a new function, flatpak_cache_http_uri() that when passed an URL and a local destination location, either a) downloads the content and stores it at the destination location, storing HTTP cache header information like Last-Modified, Etag into user xattrs (if available) or a separate file or b) if the downloaded content is already present, checks the header information to decide whether the downloaded content can be used or needs to be revalidated witha conditional request. Tests are added that use a special case test server that adds HTTP caching headers and reacts to them based on query parameters. A small test binary 'httpcache' is added for the tests to use. Closes: #1910 Approved by: alexlarsson
This commit is contained in:
Owen W. Taylor
Atomic Bot
parent
cd6a10cb66
commit
951aed561a
1
.gitignore
vendored
1
.gitignore
vendored
@ -56,6 +56,7 @@ flatpak.pc
|
||||
common/flatpak-enum-types.c
|
||||
common/flatpak-enum-types.h
|
||||
test-libflatpak
|
||||
httpcache
|
||||
Flatpak-1.0.*
|
||||
/app/parse-datetime.c
|
||||
/doc/reference/gtkdoc-check.log
|
||||
|
||||
@ -52,5 +52,14 @@ gboolean flatpak_download_http_uri (SoupSession *soup_session,
|
||||
gpointer user_data,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
gboolean flatpak_cache_http_uri (SoupSession *soup_session,
|
||||
const char *uri,
|
||||
FlatpakHTTPFlags flags,
|
||||
int dest_dfd,
|
||||
const char *dest_subpath,
|
||||
FlatpakLoadUriProgress progress,
|
||||
gpointer user_data,
|
||||
GCancellable *cancellable,
|
||||
GError **error);
|
||||
|
||||
#endif /* __FLATPAK_UTILS_HTTP_H__ */
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright © 2014 Red Hat, Inc
|
||||
* Copyright © 2018 Red Hat, Inc
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Lesser General Public
|
||||
@ -22,22 +22,307 @@
|
||||
#include "flatpak-oci-registry-private.h"
|
||||
|
||||
#include <libsoup/soup.h>
|
||||
#include "libglnx/libglnx.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/xattr.h>
|
||||
|
||||
typedef struct
|
||||
{
|
||||
char *uri;
|
||||
char *etag;
|
||||
gint64 last_modified;
|
||||
gint64 expires;
|
||||
} CacheHttpData;
|
||||
|
||||
typedef struct
|
||||
{
|
||||
GMainLoop *loop;
|
||||
GError *error;
|
||||
GOutputStream *out;
|
||||
|
||||
GOutputStream *out; /*or */
|
||||
GString *content; /* or */
|
||||
GLnxTmpfile *out_tmpfile;
|
||||
int out_tmpfile_parent_dfd;
|
||||
|
||||
guint64 downloaded_bytes;
|
||||
GString *content;
|
||||
char buffer[16 * 1024];
|
||||
FlatpakLoadUriProgress progress;
|
||||
GCancellable *cancellable;
|
||||
gpointer user_data;
|
||||
guint64 last_progress_time;
|
||||
CacheHttpData *cache_data;
|
||||
char *etag;
|
||||
} LoadUriData;
|
||||
|
||||
#define CACHE_HTTP_XATTR "user.flatpak.http"
|
||||
#define CACHE_HTTP_SUFFIX ".flatpak.http"
|
||||
#define CACHE_HTTP_TYPE "(sstt)"
|
||||
|
||||
static void
|
||||
clear_cache_http_data (CacheHttpData *data,
|
||||
gboolean clear_uri)
|
||||
{
|
||||
if (clear_uri)
|
||||
g_clear_pointer (&data->uri, g_free);
|
||||
g_clear_pointer (&data->etag, g_free);
|
||||
data->last_modified = 0;
|
||||
data->expires = 0;
|
||||
}
|
||||
|
||||
static void
|
||||
free_cache_http_data (CacheHttpData *data)
|
||||
{
|
||||
clear_cache_http_data (data, TRUE);
|
||||
g_free (data);
|
||||
}
|
||||
|
||||
G_DEFINE_AUTOPTR_CLEANUP_FUNC (CacheHttpData, free_cache_http_data)
|
||||
|
||||
static GBytes *
|
||||
serialize_cache_http_data (CacheHttpData * data)
|
||||
{
|
||||
g_autoptr(GVariant) cache_variant = NULL;
|
||||
|
||||
cache_variant = g_variant_ref_sink (g_variant_new (CACHE_HTTP_TYPE,
|
||||
data->uri,
|
||||
data->etag ? data->etag : "",
|
||||
data->last_modified,
|
||||
data->expires));
|
||||
if (G_BYTE_ORDER != G_BIG_ENDIAN)
|
||||
{
|
||||
g_autoptr(GVariant) tmp_variant = cache_variant;
|
||||
cache_variant = g_variant_byteswap (tmp_variant);
|
||||
}
|
||||
|
||||
return g_variant_get_data_as_bytes (cache_variant);
|
||||
}
|
||||
|
||||
static void
|
||||
deserialize_cache_http_data (CacheHttpData *data,
|
||||
GBytes *bytes)
|
||||
{
|
||||
g_autoptr(GVariant) cache_variant = NULL;
|
||||
|
||||
cache_variant = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE (CACHE_HTTP_TYPE),
|
||||
bytes,
|
||||
FALSE));
|
||||
if (G_BYTE_ORDER != G_BIG_ENDIAN)
|
||||
{
|
||||
g_autoptr(GVariant) tmp_variant = cache_variant;
|
||||
cache_variant = g_variant_byteswap (tmp_variant);
|
||||
}
|
||||
|
||||
g_variant_get (cache_variant,
|
||||
CACHE_HTTP_TYPE,
|
||||
&data->uri,
|
||||
&data->etag,
|
||||
&data->last_modified,
|
||||
&data->expires);
|
||||
}
|
||||
|
||||
static CacheHttpData *
|
||||
load_cache_http_data (int dfd,
|
||||
char *name,
|
||||
gboolean *no_xattr,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
g_autoptr(CacheHttpData) data = NULL;
|
||||
|
||||
g_autoptr(GBytes) cache_bytes = glnx_lgetxattrat (dfd, name,
|
||||
CACHE_HTTP_XATTR,
|
||||
error);
|
||||
if (cache_bytes == NULL)
|
||||
{
|
||||
if (errno == ENOTSUP)
|
||||
{
|
||||
g_autofree char *cache_file = NULL;
|
||||
glnx_autofd int fd = -1;
|
||||
|
||||
g_clear_error (error);
|
||||
*no_xattr = TRUE;
|
||||
|
||||
cache_file = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);
|
||||
|
||||
if (!glnx_openat_rdonly (dfd, cache_file, FALSE,
|
||||
&fd, error))
|
||||
return FALSE;
|
||||
|
||||
cache_bytes = glnx_fd_readall_bytes (fd, cancellable, error);
|
||||
if (!cache_bytes)
|
||||
return NULL;
|
||||
}
|
||||
else if (errno == ENOENT || errno == ENODATA)
|
||||
{
|
||||
g_clear_error (error);
|
||||
return g_new0 (CacheHttpData, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
data = g_new0 (CacheHttpData, 1);
|
||||
deserialize_cache_http_data (data, cache_bytes);
|
||||
return g_steal_pointer (&data);
|
||||
}
|
||||
|
||||
static void
|
||||
set_cache_http_data_from_headers (CacheHttpData *data,
|
||||
SoupMessage *msg)
|
||||
{
|
||||
const char *etag = soup_message_headers_get_one (msg->response_headers, "ETag");
|
||||
const char *last_modified = soup_message_headers_get_one (msg->response_headers, "Last-Modified");
|
||||
const char *cache_control = soup_message_headers_get_list (msg->response_headers, "Cache-Control");
|
||||
const char *expires = soup_message_headers_get_list (msg->response_headers, "Expires");
|
||||
gboolean expires_computed = FALSE;
|
||||
|
||||
/* The original HTTP 1/1 specification only required sending the ETag header in a 304
|
||||
* response, and implied that a cache might need to save the old Cache-Control
|
||||
* values. The updated RFC 7232 from 2014 requires sending Cache-Control, ETags, and
|
||||
* Expire if they would have been sent in the original 200 response, and recommends
|
||||
* sending Last-Modified for requests without an etag. Since sending these headers was
|
||||
* apparently normal previously, for simplicity we assume the RFC 7232 behavior and start
|
||||
* from scratch for a 304 response.
|
||||
*/
|
||||
clear_cache_http_data (data, FALSE);
|
||||
|
||||
if (etag && *etag)
|
||||
{
|
||||
data->etag = g_strdup (etag);
|
||||
}
|
||||
else if (last_modified && *last_modified)
|
||||
{
|
||||
SoupDate *date = soup_date_new_from_string (last_modified);
|
||||
if (date)
|
||||
{
|
||||
data->last_modified = soup_date_to_time_t (date);
|
||||
soup_date_free (date);
|
||||
}
|
||||
}
|
||||
|
||||
if (cache_control && *cache_control)
|
||||
{
|
||||
GHashTable *params = soup_header_parse_param_list (cache_control);
|
||||
GHashTableIter iter;
|
||||
gpointer key, value;
|
||||
|
||||
g_hash_table_iter_init (&iter, params);
|
||||
while (g_hash_table_iter_next (&iter, &key, &value))
|
||||
{
|
||||
if (g_strcmp0 (key, "max-age") == 0)
|
||||
{
|
||||
char *end;
|
||||
|
||||
char *max_age = value;
|
||||
int max_age_sec = g_ascii_strtoll (max_age, &end, 10);
|
||||
if (*max_age != '\0' && *end == '\0')
|
||||
{
|
||||
GTimeVal now;
|
||||
g_get_current_time (&now);
|
||||
data->expires = now.tv_sec + max_age_sec;
|
||||
expires_computed = TRUE;
|
||||
}
|
||||
}
|
||||
else if (g_strcmp0 (key, "no-cache") == 0)
|
||||
{
|
||||
data->expires = 0;
|
||||
expires_computed = TRUE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!expires_computed && expires && *expires)
|
||||
{
|
||||
SoupDate *date = soup_date_new_from_string (expires);
|
||||
if (date)
|
||||
{
|
||||
data->expires = soup_date_to_time_t (date);
|
||||
soup_date_free (date);
|
||||
expires_computed = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
if (!expires_computed)
|
||||
{
|
||||
/* If nothing implies an expires time, use 30 minutes. Browsers use
|
||||
* 0.1 * (Date - Last-Modified), but it's clearly appropriate here, and
|
||||
* better if server's send a value.
|
||||
*/
|
||||
GTimeVal now;
|
||||
g_get_current_time (&now);
|
||||
data->expires = now.tv_sec + 1800;
|
||||
}
|
||||
}
|
||||
|
||||
static gboolean
|
||||
save_cache_http_data_xattr (int fd,
|
||||
GBytes *bytes,
|
||||
GError **error)
|
||||
{
|
||||
if (TEMP_FAILURE_RETRY (fsetxattr (fd, (char *) CACHE_HTTP_XATTR,
|
||||
g_bytes_get_data (bytes, NULL),
|
||||
g_bytes_get_size (bytes),
|
||||
0)) < 0)
|
||||
return glnx_throw_errno_prefix (error, "fsetxattr");
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
save_cache_http_data_fallback (int fd,
|
||||
GBytes *bytes,
|
||||
GError **error)
|
||||
{
|
||||
if (glnx_loop_write (fd,
|
||||
g_bytes_get_data (bytes, NULL),
|
||||
g_bytes_get_size (bytes)) < 0)
|
||||
return glnx_throw_errno_prefix (error, "write");
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
save_cache_http_data_to_file (int dfd,
|
||||
char *name,
|
||||
GBytes *bytes,
|
||||
gboolean no_xattr,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
glnx_autofd int fd = -1;
|
||||
g_autofree char *fallback_name = NULL;
|
||||
|
||||
if (!no_xattr)
|
||||
{
|
||||
if (!glnx_openat_rdonly (dfd, name, FALSE,
|
||||
&fd, error))
|
||||
return FALSE;
|
||||
|
||||
if (save_cache_http_data_xattr (fd, bytes, error))
|
||||
return TRUE;
|
||||
|
||||
if (errno == ENOTSUP)
|
||||
g_clear_error (error);
|
||||
else
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);
|
||||
if (!glnx_file_replace_contents_at (dfd, fallback_name,
|
||||
g_bytes_get_data (bytes, NULL),
|
||||
g_bytes_get_size (bytes),
|
||||
0,
|
||||
cancellable,
|
||||
error))
|
||||
return FALSE;
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static void
|
||||
stream_closed (GObject *source, GAsyncResult *res, gpointer user_data)
|
||||
{
|
||||
@ -86,6 +371,15 @@ load_uri_read_cb (GObject *source, GAsyncResult *res, gpointer user_data)
|
||||
|
||||
data->downloaded_bytes += n_written;
|
||||
}
|
||||
else if (data->out_tmpfile != NULL)
|
||||
{
|
||||
if (glnx_loop_write (data->out_tmpfile->fd,
|
||||
data->buffer, nread) < 0)
|
||||
{
|
||||
glnx_throw_errno_prefix (&data->error, "write");
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
data->downloaded_bytes += nread;
|
||||
@ -130,6 +424,9 @@ load_uri_callback (GObject *source_object,
|
||||
switch (msg->status_code)
|
||||
{
|
||||
case 304:
|
||||
if (data->cache_data)
|
||||
set_cache_http_data_from_headers (data->cache_data, msg);
|
||||
|
||||
domain = FLATPAK_OCI_ERROR;
|
||||
code = FLATPAK_OCI_ERROR_NOT_CHANGED;
|
||||
break;
|
||||
@ -151,8 +448,19 @@ load_uri_callback (GObject *source_object,
|
||||
return;
|
||||
}
|
||||
|
||||
if (data->cache_data)
|
||||
set_cache_http_data_from_headers (data->cache_data, msg);
|
||||
|
||||
data->etag = g_strdup (soup_message_headers_get_one (msg->response_headers, "ETag"));
|
||||
|
||||
if (data->out_tmpfile)
|
||||
{
|
||||
if (!glnx_open_tmpfile_linkable_at (data->out_tmpfile_parent_dfd, ".",
|
||||
O_WRONLY, data->out_tmpfile,
|
||||
&data->error))
|
||||
return;
|
||||
}
|
||||
|
||||
g_input_stream_read_async (in, data->buffer, sizeof (data->buffer),
|
||||
G_PRIORITY_DEFAULT, data->cancellable,
|
||||
load_uri_read_cb, data);
|
||||
@ -286,6 +594,7 @@ flatpak_download_http_uri (SoupSession *soup_session,
|
||||
if (request == NULL)
|
||||
return FALSE;
|
||||
|
||||
m = soup_request_http_get_message (request);
|
||||
if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI)
|
||||
soup_message_headers_replace (m->request_headers, "Accept",
|
||||
"application/vnd.oci.image.manifest.v1+json");
|
||||
@ -306,3 +615,190 @@ flatpak_download_http_uri (SoupSession *soup_session,
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
static gboolean
|
||||
sync_and_rename_tmpfile (GLnxTmpfile *tmpfile,
|
||||
const char *dest_name,
|
||||
GError **error)
|
||||
{
|
||||
/* Filesystem paranoia: If we end up with the new metadata but not
|
||||
* the new data, then because the cache headers are in the metadata,
|
||||
* we'll never re-download. (If we just want to avoid losing both
|
||||
* the old and new data, skipping fdatasync when the destination is
|
||||
* missing works, but it won't here.)
|
||||
*
|
||||
* This will cause a bunch of fdatasyncs when downloading the icons for
|
||||
* a large appstream the first time, would mostly be a problem with a
|
||||
* very fast internet connection and a slow spinning drive.
|
||||
* Possible solution: update in new directory without fdatasync
|
||||
* (copying in any existing cached icons to revalidate), syncfs(), then
|
||||
* atomic symlink.
|
||||
*/
|
||||
if (fdatasync (tmpfile->fd) != 0)
|
||||
return glnx_throw_errno_prefix (error, "fdatasync");
|
||||
|
||||
if (!glnx_link_tmpfile_at (tmpfile,
|
||||
GLNX_LINK_TMPFILE_REPLACE,
|
||||
tmpfile->src_dfd, dest_name, error))
|
||||
return FALSE;
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
gboolean
|
||||
flatpak_cache_http_uri (SoupSession *soup_session,
|
||||
const char *uri,
|
||||
FlatpakHTTPFlags flags,
|
||||
int dest_dfd,
|
||||
const char *dest_subpath,
|
||||
FlatpakLoadUriProgress progress,
|
||||
gpointer user_data,
|
||||
GCancellable *cancellable,
|
||||
GError **error)
|
||||
{
|
||||
g_autoptr(SoupRequestHTTP) request = NULL;
|
||||
g_autoptr(GMainLoop) loop = NULL;
|
||||
g_autoptr(GMainContext) context = NULL;
|
||||
g_autoptr(CacheHttpData) cache_data = NULL;
|
||||
g_autofree char *parent_path = g_path_get_dirname (dest_subpath);
|
||||
g_autofree char *name = g_path_get_basename (dest_subpath);
|
||||
glnx_autofd int dfd = -1;
|
||||
gboolean no_xattr = FALSE;
|
||||
LoadUriData data = { NULL };
|
||||
g_auto(GLnxTmpfile) out_tmpfile = { 0 };
|
||||
g_auto(GLnxTmpfile) cache_tmpfile = { 0 };
|
||||
g_autoptr(GBytes) cache_bytes = NULL;
|
||||
SoupMessage *m;
|
||||
|
||||
if (!glnx_opendirat (dest_dfd, parent_path, TRUE, &dfd, error))
|
||||
return FALSE;
|
||||
|
||||
cache_data = load_cache_http_data (dfd, name, &no_xattr,
|
||||
cancellable, error);
|
||||
if (!cache_data)
|
||||
return FALSE;
|
||||
|
||||
if (g_strcmp0 (cache_data->uri, uri) != 0)
|
||||
clear_cache_http_data (cache_data, TRUE);
|
||||
|
||||
if (cache_data->uri)
|
||||
{
|
||||
GTimeVal now;
|
||||
|
||||
g_get_current_time (&now);
|
||||
if (cache_data->expires > now.tv_sec)
|
||||
{
|
||||
if (error)
|
||||
*error = g_error_new (FLATPAK_OCI_ERROR,
|
||||
FLATPAK_OCI_ERROR_NOT_CHANGED,
|
||||
"Reusing cached value");
|
||||
return FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
if (cache_data->uri == NULL)
|
||||
cache_data->uri = g_strdup (uri);
|
||||
|
||||
/* Must revalidate */
|
||||
|
||||
g_debug ("Loading %s using libsoup", uri);
|
||||
|
||||
context = g_main_context_ref_thread_default ();
|
||||
|
||||
loop = g_main_loop_new (context, TRUE);
|
||||
data.loop = loop;
|
||||
data.cache_data = cache_data;
|
||||
data.out_tmpfile = &out_tmpfile;
|
||||
data.out_tmpfile_parent_dfd = dfd;
|
||||
data.progress = progress;
|
||||
data.cancellable = cancellable;
|
||||
data.user_data = user_data;
|
||||
data.last_progress_time = g_get_monotonic_time ();
|
||||
|
||||
request = soup_session_request_http (soup_session, "GET",
|
||||
uri, error);
|
||||
if (request == NULL)
|
||||
return FALSE;
|
||||
|
||||
m = soup_request_http_get_message (request);
|
||||
|
||||
if (cache_data->etag && cache_data->etag[0])
|
||||
soup_message_headers_replace (m->request_headers, "If-None-Match", cache_data->etag);
|
||||
else if (cache_data->last_modified != 0)
|
||||
{
|
||||
SoupDate *date = soup_date_new_from_time_t (cache_data->last_modified);
|
||||
g_autofree char *date_str = soup_date_to_string (date, SOUP_DATE_HTTP);
|
||||
soup_message_headers_replace (m->request_headers, "If-Modified-Since", date_str);
|
||||
soup_date_free (date);
|
||||
}
|
||||
|
||||
if (flags & FLATPAK_HTTP_FLAGS_ACCEPT_OCI)
|
||||
soup_message_headers_replace (m->request_headers, "Accept",
|
||||
"application/vnd.oci.image.manifest.v1+json");
|
||||
|
||||
soup_request_send_async (SOUP_REQUEST (request),
|
||||
cancellable,
|
||||
load_uri_callback, &data);
|
||||
|
||||
g_main_loop_run (loop);
|
||||
|
||||
if (data.error)
|
||||
{
|
||||
if (data.error->domain == FLATPAK_OCI_ERROR &&
|
||||
data.error->code == FLATPAK_OCI_ERROR_NOT_CHANGED)
|
||||
{
|
||||
GError *tmp_error = NULL;
|
||||
|
||||
cache_bytes = serialize_cache_http_data (cache_data);
|
||||
|
||||
if (!save_cache_http_data_to_file (dfd, name, cache_bytes, no_xattr,
|
||||
cancellable, &tmp_error))
|
||||
{
|
||||
g_clear_error (&data.error);
|
||||
g_propagate_error (error, tmp_error);
|
||||
|
||||
return FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
g_propagate_error (error, data.error);
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
cache_bytes = serialize_cache_http_data (cache_data);
|
||||
if (!no_xattr)
|
||||
{
|
||||
if (!save_cache_http_data_xattr (out_tmpfile.fd, cache_bytes, error))
|
||||
{
|
||||
if (errno != ENOTSUP)
|
||||
return FALSE;
|
||||
|
||||
g_clear_error (error);
|
||||
no_xattr = TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
if (no_xattr)
|
||||
{
|
||||
if (!glnx_open_tmpfile_linkable_at (dfd, ".", O_WRONLY, &cache_tmpfile, error))
|
||||
return FALSE;
|
||||
|
||||
if (!save_cache_http_data_fallback (cache_tmpfile.fd, cache_bytes, error))
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
if (!sync_and_rename_tmpfile (&out_tmpfile, name, error))
|
||||
return FALSE;
|
||||
|
||||
if (no_xattr)
|
||||
{
|
||||
g_autofree char *fallback_name = g_strconcat (name, CACHE_HTTP_SUFFIX, NULL);
|
||||
|
||||
if (!sync_and_rename_tmpfile (&cache_tmpfile, fallback_name, error))
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
g_debug ("Received %" G_GUINT64_FORMAT " bytes", data.downloaded_bytes);
|
||||
|
||||
return TRUE;
|
||||
}
|
||||
|
||||
@ -22,6 +22,13 @@ testlibrary_LDADD = \
|
||||
$(NULL)
|
||||
testlibrary_SOURCES = tests/testlibrary.c
|
||||
|
||||
httpcache_CFLAGS = $(AM_CFLAGS) $(BASE_CFLAGS) $(OSTREE_CFLAGS) $(SOUP_CFLAGS) $(JSON_CFLAGS) $(APPSTREAM_GLIB_CFLAGS) \
|
||||
-DFLATPAK_COMPILATION \
|
||||
-DLOCALEDIR=\"$(localedir)\"
|
||||
httpcache_LDADD = $(AM_LDADD) $(BASE_LIBS) $(OSTREE_LIBS) $(SOUP_LIBS) $(JSON_LIBS) $(APPSTREAM_GLIB_LIBS) \
|
||||
libglnx.la libflatpak-common.la
|
||||
httpcache_SOURCES = tests/httpcache.c
|
||||
|
||||
tests/services/org.freedesktop.Flatpak.service: session-helper/org.freedesktop.Flatpak.service.in
|
||||
mkdir -p tests/services
|
||||
$(AM_V_GEN) $(SED) -e "s|\@libexecdir\@|$(abs_top_builddir)|" $< > $@
|
||||
@ -46,6 +53,7 @@ tests/test-basic.sh: tests/package_version.txt
|
||||
|
||||
dist_installed_test_extra_scripts += \
|
||||
buildutil/tap-driver.sh \
|
||||
tests/http-utils-test-server.py \
|
||||
tests/make-multi-collection-id-repo.sh \
|
||||
tests/make-test-app.sh \
|
||||
tests/make-test-runtime.sh \
|
||||
@ -79,6 +87,7 @@ endif
|
||||
dist_test_scripts = \
|
||||
tests/test-basic.sh \
|
||||
tests/test-build-update-repo.sh \
|
||||
tests/test-http-utils.sh \
|
||||
tests/test-run.sh \
|
||||
tests/test-run-system.sh \
|
||||
tests/test-run-deltas.sh \
|
||||
@ -96,6 +105,7 @@ dist_test_scripts = \
|
||||
$(NULL)
|
||||
|
||||
test_programs = testlibrary
|
||||
noinst_PROGRAMS += httpcache
|
||||
|
||||
@VALGRIND_CHECK_RULES@
|
||||
VALGRIND_SUPPRESSIONS_FILES=tests/flatpak.supp tests/glib.supp
|
||||
|
||||
71
tests/http-utils-test-server.py
Normal file
71
tests/http-utils-test-server.py
Normal file
@ -0,0 +1,71 @@
|
||||
#!/usr/bin/python2
|
||||
|
||||
from wsgiref.handlers import format_date_time
|
||||
from email.utils import parsedate
|
||||
from calendar import timegm
|
||||
from urlparse import parse_qs
|
||||
import BaseHTTPServer
|
||||
import time
|
||||
|
||||
server_start_time = int(time.time())
|
||||
|
||||
def parse_http_date(date):
|
||||
parsed = parsedate(date)
|
||||
if parsed is not None:
|
||||
return timegm(parsed)
|
||||
else:
|
||||
return None
|
||||
|
||||
class RequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
parts = self.path.split('?', 1)
|
||||
path = parts[0]
|
||||
if len(parts) == 1:
|
||||
query = {}
|
||||
else:
|
||||
query = parse_qs(parts[1], keep_blank_values=True)
|
||||
|
||||
response = 200
|
||||
add_headers = {}
|
||||
|
||||
if 'modified-time' in query:
|
||||
modified_since = self.headers.get("If-Modified-Since")
|
||||
if modified_since:
|
||||
modified_since_time = parse_http_date(modified_since)
|
||||
if modified_since_time <= server_start_time:
|
||||
response = 304
|
||||
add_headers["Last-Modified"] = format_date_time(server_start_time)
|
||||
|
||||
if 'etag' in query:
|
||||
etag = str(server_start_time)
|
||||
|
||||
if self.headers.get("If-None-Match") == etag:
|
||||
response = 304
|
||||
add_headers['Etag'] = etag
|
||||
|
||||
self.send_response(response)
|
||||
for k, v in add_headers.items():
|
||||
self.send_header(k, v)
|
||||
|
||||
if 'max-age' in query:
|
||||
self.send_header('Cache-Control', 'max-age=' + query['max-age'][0])
|
||||
if 'no-cache' in query:
|
||||
self.send_header('Cache-Control', 'no-cache')
|
||||
if 'expires-past' in query:
|
||||
self.send_header('Expires', format_date_time(server_start_time - 3600))
|
||||
if 'expires-future' in query:
|
||||
self.send_header('Expires', format_date_time(server_start_time + 3600))
|
||||
|
||||
if response == 200:
|
||||
self.send_header("Content-Type", "text/plain; charset=UTF-8")
|
||||
|
||||
self.end_headers()
|
||||
|
||||
if response == 200:
|
||||
self.wfile.write("path=" + self.path + "\n");
|
||||
|
||||
def test():
|
||||
BaseHTTPServer.test(RequestHandler)
|
||||
|
||||
if __name__ == '__main__':
|
||||
test()
|
||||
30
tests/httpcache.c
Normal file
30
tests/httpcache.c
Normal file
@ -0,0 +1,30 @@
|
||||
#include "common/flatpak-utils-private.h"
|
||||
|
||||
int
|
||||
main (int argc, char *argv[])
|
||||
{
|
||||
SoupSession *session = flatpak_create_soup_session (PACKAGE_STRING);
|
||||
g_autoptr(GFile) dest = NULL;
|
||||
GError *error = NULL;
|
||||
|
||||
if (argc != 3)
|
||||
{
|
||||
g_printerr("Usage testhttp URL DEST\n");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!flatpak_cache_http_uri (session,
|
||||
argv[1],
|
||||
0,
|
||||
AT_FDCWD, argv[2],
|
||||
NULL, NULL, NULL, &error))
|
||||
{
|
||||
g_print ("%s\n", error->message);
|
||||
return 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
g_print ("Server returned status 200: ok\n");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
141
tests/test-http-utils.sh
Executable file
141
tests/test-http-utils.sh
Executable file
@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright (C) 2018 Red Hat, Inc.
|
||||
#
|
||||
# This library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this library; if not, write to the
|
||||
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
||||
# Boston, MA 02111-1307, USA.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
. $(dirname $0)/libtest.sh
|
||||
|
||||
$(dirname $0)/test-webserver.sh "" "python2 $test_srcdir/http-utils-test-server.py 0"
|
||||
FLATPAK_HTTP_PID=$(cat httpd-pid)
|
||||
mv httpd-port httpd-port-main
|
||||
port=$(cat httpd-port-main)
|
||||
|
||||
assert_result() {
|
||||
test_string=$1
|
||||
remote=$2
|
||||
local=$3
|
||||
|
||||
out=`httpcache "http://localhost:$port$remote" $local || :`
|
||||
|
||||
case "$out" in
|
||||
$test_string*)
|
||||
return
|
||||
;;
|
||||
*)
|
||||
echo "For $remote => $local, expected '$test_string', got '$out'"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
assert_cached() {
|
||||
assert_result "Reusing cached value" $@
|
||||
}
|
||||
|
||||
assert_304() {
|
||||
assert_result "Server returned status 304:" $@
|
||||
}
|
||||
|
||||
assert_ok() {
|
||||
assert_result "Server returned status 200:" $@
|
||||
}
|
||||
|
||||
|
||||
have_xattrs() {
|
||||
touch $1/test-xattrs
|
||||
setfattr -n user.testvalue -v somevalue $1/test-xattrs > /dev/null 2>&1
|
||||
}
|
||||
|
||||
echo "1..4"
|
||||
|
||||
# Without anything else, cached for 30 minutes
|
||||
assert_ok "/" $test_tmpdir/output
|
||||
assert_cached "/" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
# An explicit cache-lifetime
|
||||
assert_ok "/?max-age=3600" $test_tmpdir/output
|
||||
assert_cached "/?max-age=3600" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
# Turn off caching
|
||||
assert_ok "/?max-age=0" $test_tmpdir/output
|
||||
assert_ok "/?max-age=0" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
# Turn off caching a different way
|
||||
assert_ok "/?no-cache" $test_tmpdir/output
|
||||
assert_ok "/?no-cache" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
# Expires support
|
||||
assert_ok "/?expires-future" $test_tmpdir/output
|
||||
assert_cached "/?expires-future" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
assert_ok "/?expires-past" $test_tmpdir/output
|
||||
assert_ok "/?expires-past" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
echo 'ok http cache lifetimes'
|
||||
|
||||
# Revalation with an etag
|
||||
assert_ok "/?etag&no-cache" $test_tmpdir/output
|
||||
assert_304 "/?etag&no-cache" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
# Revalation with an modified time
|
||||
assert_ok "/?modified-time&no-cache" $test_tmpdir/output
|
||||
assert_304 "/?modified-time&no-cache" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
|
||||
echo 'ok http revalidation'
|
||||
|
||||
# Testing that things work with without xattr support
|
||||
|
||||
if have_xattrs $test_tmpdir ; then
|
||||
ls $test_tmpdir 1>&2
|
||||
assert_ok "/?etag&no-cache" $test_tmpdir/output
|
||||
ls $test_tmpdir 1>&2
|
||||
assert_not_has_file $test_tmpdir/output.flatpak.http
|
||||
assert_304 "/?etag&no-cache" $test_tmpdir/output
|
||||
rm -f $test_tmpdir/output*
|
||||
echo "ok with-xattrs"
|
||||
else
|
||||
echo "ok with-xattrs # skip /var/tmp doesn't have user xattr support"
|
||||
fi
|
||||
|
||||
# Testing fallback without xattr support
|
||||
|
||||
no_xattrs_tempdir=`mktemp -d /tmp/test-flatpak-XXXXXX`
|
||||
no_xattrs_cleanup () {
|
||||
rm -rf test_tmpdir
|
||||
cleanup
|
||||
}
|
||||
trap no_xattrs_cleanup EXIT
|
||||
|
||||
if have_xattrs $no_xattrs_tempdir ; then
|
||||
echo "ok no-xattrs # skip /tmp has user xattr support"
|
||||
else
|
||||
assert_ok "/?etag&no-cache" $no_xattrs_tempdir/output
|
||||
assert_has_file $no_xattrs_tempdir/output.flatpak.http
|
||||
assert_304 "/?etag&no-cache" $no_xattrs_tempdir/output
|
||||
rm -f $no_xattrs_tempdir/output*
|
||||
echo "ok no-xattrs"
|
||||
fi
|
||||
@ -3,10 +3,11 @@
|
||||
set -euo pipefail
|
||||
|
||||
dir=$1
|
||||
cmd=${2:-python -m SimpleHTTPServer 0}
|
||||
test_tmpdir=$(pwd)
|
||||
|
||||
cd ${dir}
|
||||
env PYTHONUNBUFFERED=1 setsid python -m SimpleHTTPServer 0 >${test_tmpdir}/httpd-output &
|
||||
[ "$dir" != "" ] && cd ${dir}
|
||||
env PYTHONUNBUFFERED=1 setsid $cmd >${test_tmpdir}/httpd-output &
|
||||
child_pid=$!
|
||||
echo "Web server pid: $child_pid" >&2
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user