Skip to content

CIRCLE / cloud

Go to a project
Commit 4816e34a by Őry Máté
Options

store: rewrite

parent e98b628f
Showing with 173 additions and 234 deletions
circle/dashboard/models.py
...@@ -44,7 +44,7 @@ from common.models import HumanReadableObject, create_readable, Encoder ...@@ -44,7 +44,7 @@ from common.models import HumanReadableObject, create_readable, Encoder
from vm.tasks.agent_tasks import add_keys, del_keys from vm.tasks.agent_tasks import add_keys, del_keys
from dashboard import store_api from .store_api import Store
logger = getLogger(__name__) logger = getLogger(__name__)
...@@ -213,12 +213,10 @@ def create_profile(sender, user, request, **kwargs): ...@@ -213,12 +213,10 @@ def create_profile(sender, user, request, **kwargs):
return False return False
profile, created = Profile.objects.get_or_create(user=user) profile, created = Profile.objects.get_or_create(user=user)
if created: try:
user_home = "u-%d" % user.pk Store(user).create_user(profile.smb_password, None, profile.disk_quota)
if not store_api.userexist(user_home): except:
store_api.createuser(user_home, profile.smb_password, None, logger.exception("Can't create user %s", unicode(user))
profile.disk_quota)
return created return created
user_logged_in.connect(create_profile) user_logged_in.connect(create_profile)
......
circle/dashboard/store_api.py
from django.http import Http404
import json import json
import logging import logging
import requests from urlparse import urljoin
from datetime import datetime from datetime import datetime
from sizefield.utils import filesizeformat
from django.http import Http404
from django.conf import settings from django.conf import settings
from requests import get, post, codes
from sizefield.utils import filesizeformat
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class Mock(object): class StoreApiException(Exception):
pass pass
def get_host(): class NotOkException(StoreApiException):
return settings.STORE_URL def __init__(self, status, *args, **kwargs):
self.status = status
super(NotOkException, self).__init__(*args, **kwargs)
def get_request_arguments():
args = {'verify': settings.STORE_VERIFY_SSL}
class Store(object):
if settings.STORE_SSL_AUTH:
args['cert'] = (settings.STORE_CLIENT_CERT, settings.STORE_CLIENT_KEY) def __init__(self, user, default_timeout=0.5):
if settings.STORE_BASIC_AUTH: self.request_args = {'verify': settings.STORE_VERIFY_SSL}
args['auth'] = (settings.STORE_CLIENT_USER, if settings.STORE_SSL_AUTH:
settings.STORE_CLIENT_PASSWORD) self.request_args['cert'] = (settings.STORE_CLIENT_CERT,
return args settings.STORE_CLIENT_KEY)
if settings.STORE_BASIC_AUTH:
self.request_args['auth'] = (settings.STORE_CLIENT_USER,
def post_request(url, payload, timeout=None): settings.STORE_CLIENT_PASSWORD)
try: self.username = "u-%d" % user.pk
headers = {'content-type': 'application/json'} self.default_timeout = default_timeout
r = requests.post(url, data=payload, headers=headers, timeout=timeout, self.store_url = settings.STORE_URL
**get_request_arguments())
return r def _request(self, url, method=get, timeout=None,
except Exception as e: raise_status_code=True, **kwargs):
logger.error("Error in store POST: %s" % e) url = urljoin(self.store_url, url)
dummy = Mock() if timeout is None:
setattr(dummy, "status_code", 200) timeout = self.default_timeout
setattr(dummy, "content", "[]") payload = json.dumps(kwargs)
return dummy try:
headers = {'content-type': 'application/json'}
response = method(url, data=payload, headers=headers,
def get_request(url, timeout=None): timeout=timeout, **self.request_args)
try: except Exception:
headers = {'content-type': 'application/json'} logger.exception("Error in store %s loading %s",
r = requests.get(url, headers=headers, timeout=timeout, unicode(method), url)
**get_request_arguments()) raise
return r
except Exception as e:
logger.error("Error in store GET: %s" % e)
dummy = Mock()
setattr(dummy, "status_code", 200)
setattr(dummy, "content", "[]")
return dummy
def listfolder(neptun, path):
url = settings.STORE_URL + '/' + neptun
payload = json.dumps({'CMD': 'LIST', 'PATH': path})
r = post_request(url, payload, timeout=5)
if r.status_code == requests.codes.ok:
tupplelist = json.loads(r.content)
return tupplelist
else:
raise Http404
def toplist(neptun):
url = settings.STORE_URL + '/' + neptun
payload = json.dumps({'CMD': 'TOPLIST'})
r = post_request(url, payload, timeout=2)
if r.status_code == requests.codes.ok:
tupplelist = json.loads(r.content)
return tupplelist
else:
raise Http404
def requestdownload(neptun, path):
url = settings.STORE_URL + '/' + neptun
payload = json.dumps({'CMD': 'DOWNLOAD', 'PATH': path})
r = post_request(url, payload)
response = json.loads(r.content)
return response['LINK']
def requestupload(neptun, path):
url = settings.STORE_URL+'/'+neptun
payload = json.dumps({'CMD': 'UPLOAD', 'PATH': path})
r = post_request(url, payload)
response = json.loads(r.content)
print response
if r.status_code == requests.codes.ok:
return response['LINK']
else:
raise Http404
def requestremove(neptun, path):
url = settings.STORE_URL+'/'+neptun
payload = json.dumps({'CMD': 'REMOVE', 'PATH': path})
r = post_request(url, payload)
if r.status_code == requests.codes.ok:
return True
else:
return False
def requestnewfolder(neptun, path):
url = settings.STORE_URL+'/'+neptun
payload = json.dumps({'CMD': 'NEW_FOLDER', 'PATH': path})
r = post_request(url, payload)
if r.status_code == requests.codes.ok:
return True
else:
return False
def requestrename(neptun, old_path, new_name):
url = settings.STORE_URL+'/'+neptun
payload = json.dumps(
{'CMD': 'RENAME', 'NEW_NAME': new_name, 'PATH': old_path})
r = post_request(url, payload)
if r.status_code == requests.codes.ok:
return True
else:
return False
def requestquota(neptun):
url = settings.STORE_URL+'/'+neptun
r = get_request(url)
if r.status_code == requests.codes.ok:
return json.loads(r.content)
else:
return False
def set_quota(neptun, quota):
url = settings.STORE_URL+'/quota/'+neptun
payload = json.dumps({'QUOTA': quota})
r = post_request(url, payload)
if r.status_code == requests.codes.ok:
return True
else:
return False
def userexist(neptun):
url = settings.STORE_URL+'/'+neptun
r = get_request(url, timeout=5)
if r.status_code == requests.codes.ok:
return True
else:
return False
def createuser(neptun, password, key_list, quota):
url = settings.STORE_URL+'/new/'+neptun
payload = json.dumps(
{'SMBPASSWD': password, 'KEYS': key_list, 'QUOTA': quota})
r = post_request(url, payload, timeout=5)
if r.status_code == requests.codes.ok:
return True
else:
return False
def updateauthorizationinfo(neptun, password, key_list):
url = settings.STORE_URL+'/set/'+neptun
payload = json.dumps({'SMBPASSWD': password, 'KEYS': key_list})
r = post_request(url, payload)
if r.status_code == requests.codes.ok:
return True
else:
return False
def process_list(content):
for d in content:
d['human_readable_date'] = datetime.utcfromtimestamp(float(
d['MTIME']))
delta = (datetime.utcnow() - d['human_readable_date']).total_seconds()
d['is_new'] = delta < 5 and delta > 0
d['human_readable_size'] = (
"directory" if d['TYPE'] == "D" else
filesizeformat(float(d['SIZE'])))
if len(d['DIR']) == 1 and d['DIR'][0] == ".":
d['directory'] = "/"
else: else:
d['directory'] = "/" + d['DIR'] + "/" if raise_status_code and response.status_code != codes.ok:
if response.status_code == 404:
d['path'] = d['directory'] raise Http404()
d['path'] += d['NAME'] else:
if d['TYPE'] == "D": raise NotOkException(response.status_code)
d['path'] += "/" return response
def _request_cmd(self, cmd, **kwargs):
return self._request(self.username, post, CMD=cmd)
def list(self, path, process=True):
r = self._request_cmd("LIST", PATH=path)
result = r.json()
if process:
return self._process_list(result)
else:
return result
return sorted(content, key=lambda k: k['TYPE']) def toplist(self, process=True):
r = self._request_cmd("TOPLIST")
result = r.json()
if process:
return self._process_list(result)
else:
return result
def request_download(self, path):
r = self._request_cmd("DOWNLOAD", PATH=path)
return r.json()['LINK']
def request_upload(self, path):
r = self._request_cmd("UPLOAD", PATH=path)
return r.json()['LINK']
def remove(self, path):
self._request_cmd("REMOVE", PATH=path)
def new_folder(self, path):
self._request_cmd("NEW_FOLDER", PATH=path)
def rename(self, old_path, new_name):
self._request_cmd("RENAME", PATH=old_path, NEW_NAME=new_name)
def get_quota(self): # no CMD? :o
r = self._request(self.username)
return r.json()
def set_quota(self, quota):
self._request(self.username + "/quota/", post, QUOTA=quota)
def user_exist(self):
try:
self._request(self.username)
return True
except NotOkException:
return False
def create_user(self, password, keys, quota):
self._request("/new/" + self.username, SMBPASSWD=password, KEYS=keys,
QUOTA=quota)
@staticmethod
def _process_list(content):
for d in content:
d['human_readable_date'] = datetime.utcfromtimestamp(float(
d['MTIME']))
delta = (datetime.utcnow() -
d['human_readable_date']).total_seconds()
d['is_new'] = 0 < delta < 5
d['human_readable_size'] = (
"directory" if d['TYPE'] == "D" else
filesizeformat(float(d['SIZE'])))
if len(d['DIR']) == 1 and d['DIR'][0] == ".":
d['directory'] = "/"
else:
d['directory'] = "/" + d['DIR'] + "/"
d['path'] = d['directory']
d['path'] += d['NAME']
if d['TYPE'] == "D":
d['path'] += "/"
return sorted(content, key=lambda k: k['TYPE'])
circle/dashboard/views.py
...@@ -21,6 +21,7 @@ from __future__ import unicode_literals, absolute_import ...@@ -21,6 +21,7 @@ from __future__ import unicode_literals, absolute_import
from collections import OrderedDict from collections import OrderedDict
from itertools import chain from itertools import chain
from os import getenv from os import getenv
from os.path import join, normpath, dirname
import os import os
import json import json
import logging import logging
...@@ -83,7 +84,7 @@ from storage.models import Disk ...@@ -83,7 +84,7 @@ from storage.models import Disk
from firewall.models import Vlan, Host, Rule from firewall.models import Vlan, Host, Rule
from .models import Favourite, Profile, GroupProfile, FutureMember from .models import Favourite, Profile, GroupProfile, FutureMember
from dashboard import store_api from .store_api import Store
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
saml_available = hasattr(settings, "SAML_CONFIG") saml_available = hasattr(settings, "SAML_CONFIG")
...@@ -225,14 +226,15 @@ class IndexView(LoginRequiredMixin, TemplateView): ...@@ -225,14 +226,15 @@ class IndexView(LoginRequiredMixin, TemplateView):
'operator', user).all()[:5] 'operator', user).all()[:5]
# toplist # toplist
user_home = "u-%d" % user.pk cache_key = "toplist-%d" % self.request.user.pk
cache_key = "toplist-%s" % user_home
cache = get_cache("default") cache = get_cache("default")
toplist = cache.get(cache_key) toplist = cache.get(cache_key)
if not toplist: if not toplist:
try: try:
toplist = store_api.process_list(store_api.toplist(user_home)) toplist = Store(self.request.user).toplist()
except Http404: except Exception:
logger.exception("Unable to get tolist for %s",
unicode(self.request.user))
toplist = [] toplist = []
cache.set(cache_key, toplist, 300) cache.set(cache_key, toplist, 300)
...@@ -3092,14 +3094,12 @@ class StoreList(LoginRequiredMixin, TemplateView): ...@@ -3092,14 +3094,12 @@ class StoreList(LoginRequiredMixin, TemplateView):
directory = self.request.GET.get("directory", "/") directory = self.request.GET.get("directory", "/")
directory = "/" if not len(directory) else directory directory = "/" if not len(directory) else directory
user_home = "u-%d" % self.request.user.pk context['root'] = Store(self.request.user).list(directory)
content = store_api.listfolder(user_home, directory)
context['root'] = store_api.process_list(content)
context['up_url'] = self.create_up_directory(directory) context['up_url'] = self.create_up_directory(directory)
context['current'] = directory context['current'] = directory
context['next_url'] = "%s%s?directory=%s" % ( context['next_url'] = "%s%s?directory=%s" % (
settings.DJANGO_URL[:-1], reverse("dashboard.views.store-list"), settings.DJANGO_URL.rstrip("/"),
directory) reverse("dashboard.views.store-list"), directory)
return context return context
def get(self, *args, **kwargs): def get(self, *args, **kwargs):
...@@ -3113,28 +3113,31 @@ class StoreList(LoginRequiredMixin, TemplateView): ...@@ -3113,28 +3113,31 @@ class StoreList(LoginRequiredMixin, TemplateView):
return super(StoreList, self).get(*args, **kwargs) return super(StoreList, self).get(*args, **kwargs)
def create_up_directory(self, directory): def create_up_directory(self, directory):
cut = -2 if directory.endswith("/") else -1 return normpath(join('/', directory, '..'))
return "/".join(directory.split("/")[:cut]) + "/"
@require_GET @require_GET
@login_required @login_required
def store_download(request): def store_download(request):
user_home = "u-%d" % request.user.pk
path = request.GET.get("path") path = request.GET.get("path")
url = store_api.requestdownload(user_home, path) url = Store(request.user).request_download(path)
return redirect(url) return redirect(url)
@require_GET @require_GET
@login_required @login_required
def store_upload(request): def store_upload(request):
user_home = "u-%d" % request.user.pk
directory = request.GET.get("directory", "/") directory = request.GET.get("directory", "/")
action = store_api.requestupload(user_home, directory) try:
action = Store(request.user).request_upload(directory)
except Exception:
logger.exception("Unable to upload")
messages.error(request, _("Unable to upload file."))
return redirect("/")
next_url = "%s%s?directory=%s" % ( next_url = "%s%s?directory=%s" % (
settings.DJANGO_URL[:-1], reverse("dashboard.views.store-list"), settings.DJANGO_URL.rstrip("/"),
directory) reverse("dashboard.views.store-list"), directory)
return render(request, "dashboard/store/upload.html", return render(request, "dashboard/store/upload.html",
{'directory': directory, 'action': action, {'directory': directory, 'action': action,
...@@ -3144,9 +3147,13 @@ def store_upload(request): ...@@ -3144,9 +3147,13 @@ def store_upload(request):
@require_GET @require_GET
@login_required @login_required
def store_get_upload_url(request): def store_get_upload_url(request):
user_home = "u-%d" % request.user.pk
current_dir = request.GET.get("current_dir") current_dir = request.GET.get("current_dir")
url = store_api.requestupload(user_home, current_dir) try:
url = Store(request.user).request_upload(current_dir)
except Exception:
logger.exception("Unable to upload")
messages.error(request, _("Unable to upload file."))
return redirect("/")
return HttpResponse( return HttpResponse(
json.dumps({'url': url}), content_type="application/json") json.dumps({'url': url}), content_type="application/json")
...@@ -3171,30 +3178,31 @@ class StoreRemove(LoginRequiredMixin, TemplateView): ...@@ -3171,30 +3178,31 @@ class StoreRemove(LoginRequiredMixin, TemplateView):
return context return context
def post(self, *args, **kwargs): def post(self, *args, **kwargs):
user_home = "u-%d" % self.request.user.pk
path = self.request.POST.get("path") path = self.request.POST.get("path")
store_api.requestremove(user_home, path) try:
Store(self.request.user).remove(path)
except Exception:
logger.exception("Unable to remove %s", path)
messages.error(self.request, _("Unable to remove %s.") % path)
if path.endswith("/"): return redirect("%s?directory=%s" % (
return redirect("%s?directory=%s" % ( reverse("dashboard.views.store-list"),
reverse("dashboard.views.store-list"), dirname(dirname(path)),
os.path.dirname(os.path.dirname(path)), ))
))
else:
return redirect("%s?directory=%s" % (
reverse("dashboard.views.store-list"),
os.path.dirname(path),
))
@require_POST @require_POST
@login_required @login_required
def store_new_directory(request): def store_new_directory(request):
user_home = "u-%d" % request.user.pk
path = request.POST.get("path") path = request.POST.get("path")
name = request.POST.get("name") name = request.POST.get("name")
store_api.requestnewfolder(user_home, path + name) try:
Store(request.user).new_folder(join(path, name))
except Exception:
logger.exception("Unable to create folder %s in %s for %s",
name, path, unicode(request.user))
messages.error(request, _("Unable to create folder."))
return redirect("%s?directory=%s" % ( return redirect("%s?directory=%s" % (
reverse("dashboard.views.store-list"), path)) reverse("dashboard.views.store-list"), path))
...@@ -3202,12 +3210,12 @@ def store_new_directory(request): ...@@ -3202,12 +3210,12 @@ def store_new_directory(request):
@require_POST @require_POST
@login_required @login_required
def store_refresh_toplist(request): def store_refresh_toplist(request):
user_home = "u-%d" % request.user.pk cache_key = "toplist-%d" % request.user.pk
cache_key = "toplist-%s" % user_home
cache = get_cache("default") cache = get_cache("default")
try: try:
toplist = store_api.process_list(store_api.toplist(user_home)) toplist = Store(request.user).toplist()
except Http404: except Exception:
logger.exception("Can't get toplist of %s", unicode(request.user))
toplist = [] toplist = []
cache.set(cache_key, toplist, 300) cache.set(cache_key, toplist, 300)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment