1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 from twisted.internet import defer
19
20 from flumotion.common import log
21 from flumotion.component.misc.httpserver import cachemanager
22 from flumotion.component.misc.httpserver import cachestats
23 from flumotion.component.misc.httpserver import localpath
24 from flumotion.component.misc.httpserver.httpcached import http_client
25 from flumotion.component.misc.httpserver.httpcached import http_utils
26 from flumotion.component.misc.httpserver.httpcached import request_manager
27 from flumotion.component.misc.httpserver.httpcached import resource_manager
28 from flumotion.component.misc.httpserver.httpcached import server_selection
29 from flumotion.component.misc.httpserver.httpcached import strategy_basic
30
31
32 LOG_CATEGORY = "filereader-httpcached"
33
34 DEFAULT_CACHE_TTL = 5*60
35 DEFAULT_DNS_REFRESH = 60
36 DEFAULT_VIRTUAL_PORT = 80
37 DEFAULT_VIRTUAL_PATH = ""
38 DEFAULT_SERVER_PORT = 3128
39 DEFAULT_PROXY_PRIORITY = 1
40 DEFAULT_CONN_TIMEOUT = 2
41 DEFAULT_IDLE_TIMEOUT = 5
42
43
45 """
46 Offers a file-like interface to streams retrieved using HTTP.
47 It supports:
48 - Local caching with TTL expiration, and cooperative managment.
49 - Load-balanced HTTP servers with priority level (fall-back).
50 - More than one IP by server hostname with periodic DNS refresh.
51 - Connection resuming if HTTP connection got disconnected.
52 """
53
54 logCategory = LOG_CATEGORY
55
57 props = args['properties']
58
59 cacheDir = props.get('cache-dir')
60 cacheSizeInMB = props.get('cache-size')
61 if cacheSizeInMB is not None:
62 cacheSize = cacheSizeInMB * 10 ** 6
63 else:
64 cacheSize = None
65 cleanupEnabled = props.get('cleanup-enabled')
66 cleanupHighWatermark = props.get('cleanup-high-watermark')
67 cleanupLowWatermark = props.get('cleanup-low-watermark')
68
69 self.virtualHost = props.get('virtual-hostname')
70 self.virtualPort = props.get('virtual-port', DEFAULT_VIRTUAL_PORT)
71 self.virtualPath = props.get('virtual-path', DEFAULT_VIRTUAL_PATH)
72 dnsRefresh = props.get('dns-refresh-period', DEFAULT_DNS_REFRESH)
73 servers = props.get('http-server')
74 compat_servers = props.get('http-server-old')
75
76 self.stats = cachestats.CacheStatistics()
77
78 self.cachemgr = cachemanager.CacheManager(self.stats,
79 cacheDir, cacheSize,
80 cleanupEnabled,
81 cleanupHighWatermark,
82 cleanupLowWatermark,
83 self.virtualHost)
84
85 selector = server_selection.ServerSelector(dnsRefresh)
86
87 if not (servers or compat_servers):
88 selector.addServer(self.virtualHost, self.virtualPort)
89 else:
90 if compat_servers:
91
92 for hostname in compat_servers:
93 if '#' in hostname:
94 hostname, priostr = hostname.split('#', 1)
95 priority = int(priostr)
96 else:
97 priority = DEFAULT_PROXY_PRIORITY
98 if ':' in hostname:
99 hostname, portstr = hostname.split(':', 1)
100 port = int(portstr)
101 else:
102 port = DEFAULT_SERVER_PORT
103 selector.addServer(hostname, port, priority)
104
105
106 if servers:
107
108 for serverProps in servers:
109 hostname = serverProps.get('hostname')
110 port = serverProps.get('port', DEFAULT_SERVER_PORT)
111 priority = serverProps.get('priority',
112 DEFAULT_PROXY_PRIORITY)
113 selector.addServer(hostname, port, priority)
114
115 connTimeout = props.get('connection-timeout', DEFAULT_CONN_TIMEOUT)
116 idleTimeout = props.get('idle-timeout', DEFAULT_IDLE_TIMEOUT)
117
118 client = http_client.StreamRequester(connTimeout, idleTimeout)
119
120 reqmgr = request_manager.RequestManager(selector, client)
121
122 cacheTTL = props.get('cache-ttl', DEFAULT_CACHE_TTL)
123
124 self.strategy = strategy_basic.CachingStrategy(self.cachemgr,
125 reqmgr, cacheTTL)
126
127 self.resmgr = resource_manager.ResourceManager(self.strategy,
128 self.stats)
129
131 d = defer.Deferred()
132 d.addCallback(lambda _: self.cachemgr.setUp())
133 d.addCallback(lambda _: self.strategy.setup())
134 d.addCallback(lambda _: self)
135 d.callback(None)
136 return d
137
144
145 - def open(self, path):
150