Commit 44c50b76 by Máhonfai Bálint

Implement disk export with plain SCP instead of paramiko

Paramiko and other tested Python SSH libraries are all very slow compared to the scp command, so implement this feature without a library.
parent 73d6473c
...@@ -7,7 +7,6 @@ from zipfile import ZipFile, is_zipfile ...@@ -7,7 +7,6 @@ from zipfile import ZipFile, is_zipfile
import magic import magic
import os import os
import paramiko
import re import re
import requests import requests
from bz2 import BZ2Decompressor from bz2 import BZ2Decompressor
...@@ -322,7 +321,7 @@ class Disk(object): ...@@ -322,7 +321,7 @@ class Disk(object):
self.size = Disk.get(self.dir, self.name).size self.size = Disk.get(self.dir, self.name).size
def export(self, task, disk_format, exported_name, upload_link, parent_id): def export(self, task, disk_format, upload_link, port=None):
exported_path = self.get_path() + '.' + disk_format exported_path = self.get_path() + '.' + disk_format
cmdline = ['ionice', '-c', 'idle', cmdline = ['ionice', '-c', 'idle',
'qemu-img', 'convert'] 'qemu-img', 'convert']
...@@ -331,49 +330,26 @@ class Disk(object): ...@@ -331,49 +330,26 @@ class Disk(object):
cmdline.extend(['-m', '4', '-O', disk_format, cmdline.extend(['-m', '4', '-O', disk_format,
self.get_path(), self.get_path(),
exported_path]) exported_path])
subprocess.check_output(cmdline) subprocess.check_output(cmdline)
size = os.path.getsize(exported_path)
percent = [0] cmdline = ['scp', '-B']
if port is not None:
cmdline.extend(['-P', str(port)])
cmdline.extend([exported_path, upload_link])
def update_state(monitor): proc = subprocess.Popen(cmdline)
new_percent = monitor.bytes_read * 100 / size try:
if new_percent > percent[0]: while proc.poll() is None:
if task.is_aborted(): if task.is_aborted():
raise AbortException() raise AbortException()
percent[0] = new_percent sleep(1)
task.update_state( except AbortException:
task_id=parent_id, proc.terminate()
state=task.AsyncResult(parent_id).state, logger.info("Export of disk %s aborted" % self.name)
meta={'size': monitor.bytes_read, 'percent': percent[0]} finally:
) os.unlink(exported_path)
with open(exported_path, 'rb') as exported_disk: return os.path.basename(exported_path)
try:
e = MultipartEncoder(
{'data': (exported_name + '.' + disk_format,
exported_disk,
'application/octet-stream')}
)
m = MultipartEncoderMonitor(e, update_state)
# Force the read function to read more than 8192 bytes,
# which is a hardcoded value in httplib. This increases
# the upload speed. See:
# https://github.com/requests/toolbelt/issues/75
m._read = m.read
m.read = lambda _: m._read(1024 * 1024)
response = requests.post(
upload_link,
data=m,
headers={'Content-Type': m.content_type},
params={'no_redirect': ''}
)
if response.status_code != 200:
raise Exception("Invalid response status code: %s" %
response.status_code)
finally:
os.unlink(exported_path)
def extract_iso_from_zip(self, disk_path): def extract_iso_from_zip(self, disk_path):
with ZipFile(disk_path, 'r') as z: with ZipFile(disk_path, 'r') as z:
......
...@@ -63,11 +63,13 @@ class export_disk(AbortableTask): ...@@ -63,11 +63,13 @@ class export_disk(AbortableTask):
def run(self, **kwargs): def run(self, **kwargs):
disk_desc = kwargs["disk_desc"] disk_desc = kwargs["disk_desc"]
disk_format = kwargs["disk_format"] disk_format = kwargs["disk_format"]
exported_name = kwargs["exported_name"]
upload_link = kwargs["upload_link"] upload_link = kwargs["upload_link"]
parent_id = kwargs["task"] if "port" in kwargs:
port = kwargs["port"]
else:
port = None
disk = Disk.deserialize(disk_desc) disk = Disk.deserialize(disk_desc)
disk.export(self, disk_format, exported_name, upload_link, parent_id) return disk.export(self, disk_format, upload_link, port)
@celery.task() @celery.task()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment