Skip to content
Toggle navigation
P
Projects
G
Groups
S
Snippets
Help
CIRCLE
/
cloud
This project
Loading...
Sign in
Toggle navigation
Go to a project
Project
Repository
Issues
94
Merge Requests
10
Pipelines
Wiki
Snippets
Members
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit
e60ac87d
authored
Mar 17, 2014
by
Bach Dániel
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'issue-105' into 'master'
Change Disk.ready
#105
parents
1bd118f7
1091dd05
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
182 additions
and
36 deletions
+182
-36
circle/dashboard/fixtures/test-vm-fixture.json
+0
-1
circle/dashboard/tests/test_views.py
+3
-2
circle/manager/scheduler.py
+21
-2
circle/storage/migrations/0013_auto__del_field_disk_ready.py
+115
-0
circle/storage/models.py
+43
-31
No files found.
circle/dashboard/fixtures/test-vm-fixture.json
View file @
e60ac87d
...
...
@@ -35,7 +35,6 @@
"filename"
:
"disc.img"
,
"destroyed"
:
null
,
"base"
:
null
,
"ready"
:
true
,
"datastore"
:
1
,
"dev_num"
:
"a"
,
"type"
:
"qcow2-norm"
,
...
...
circle/dashboard/tests/test_views.py
View file @
e60ac87d
...
...
@@ -278,7 +278,7 @@ class VmDetailTest(LoginMixin, TestCase):
self
.
login
(
c
,
"user1"
)
inst
=
Instance
.
objects
.
get
(
pk
=
1
)
inst
.
set_level
(
self
.
u1
,
'owner'
)
disks
=
inst
.
disks
.
count
()
#
disks = inst.disks.count()
response
=
c
.
post
(
"/dashboard/disk/add/"
,
{
'disk-name'
:
"a"
,
'disk-size'
:
1
,
...
...
@@ -286,7 +286,8 @@ class VmDetailTest(LoginMixin, TestCase):
'disk-object_pk'
:
1
,
})
self
.
assertEqual
(
response
.
status_code
,
302
)
self
.
assertEqual
(
disks
+
1
,
inst
.
disks
.
count
())
# mancelery is needed TODO
# self.assertEqual(disks + 1, inst.disks.count())
def
test_notification_read
(
self
):
c
=
Client
()
...
...
circle/manager/scheduler.py
View file @
e60ac87d
from
logging
import
getLogger
from
django.db.models
import
Sum
logger
=
getLogger
(
__name__
)
class
NotEnoughMemoryException
(
Exception
):
...
...
@@ -24,20 +28,25 @@ def select_node(instance, nodes):
'''
# check required traits
nodes
=
[
n
for
n
in
nodes
if
n
.
enabled
and
has_traits
(
instance
.
req_traits
.
all
(),
n
)]
if
n
.
enabled
and
n
.
online
and
has_traits
(
instance
.
req_traits
.
all
(),
n
)]
if
not
nodes
:
logger
.
warning
(
'select_node: no usable node for
%
s'
,
unicode
(
instance
))
raise
TraitsUnsatisfiableException
()
# check required RAM
nodes
=
[
n
for
n
in
nodes
if
has_enough_ram
(
instance
.
ram_size
,
n
)]
if
not
nodes
:
logger
.
warning
(
'select_node: no enough RAM for
%
s'
,
unicode
(
instance
))
raise
NotEnoughMemoryException
()
# sort nodes first by processor usage, then priority
nodes
.
sort
(
key
=
lambda
n
:
n
.
priority
,
reverse
=
True
)
nodes
.
sort
(
key
=
free_cpu_time
,
reverse
=
True
)
result
=
nodes
[
0
]
return
nodes
[
0
]
logger
.
info
(
'select_node:
%
s for
%
s'
,
unicode
(
result
),
unicode
(
instance
))
return
result
def
has_traits
(
traits
,
node
):
...
...
@@ -51,6 +60,7 @@ def has_enough_ram(ram_size, node):
"""True, if the node has enough memory to accomodate a guest requiring
ram_size mebibytes of memory; otherwise, false.
"""
try
:
total
=
node
.
ram_size
used
=
(
node
.
ram_usage
/
100
)
*
total
unused
=
total
-
used
...
...
@@ -60,6 +70,10 @@ def has_enough_ram(ram_size, node):
free
=
overcommit
-
reserved
return
ram_size
<
unused
and
ram_size
<
free
except
TypeError
as
e
:
logger
.
warning
(
'Got incorrect monitoring data for node
%
s.
%
s'
,
unicode
(
node
),
unicode
(
e
))
return
False
def
free_cpu_time
(
node
):
...
...
@@ -67,7 +81,12 @@ def free_cpu_time(node):
Higher values indicate more idle time.
"""
try
:
activity
=
node
.
cpu_usage
/
100
inactivity
=
1
-
activity
cores
=
node
.
num_cores
return
cores
*
inactivity
except
TypeError
as
e
:
logger
.
warning
(
'Got incorrect monitoring data for node
%
s.
%
s'
,
unicode
(
node
),
unicode
(
e
))
return
False
# monitoring data is incorrect
circle/storage/migrations/0013_auto__del_field_disk_ready.py
0 → 100644
View file @
e60ac87d
# -*- coding: utf-8 -*-
import
datetime
from
south.db
import
db
from
south.v2
import
SchemaMigration
from
django.db
import
models
class
Migration
(
SchemaMigration
):
def
forwards
(
self
,
orm
):
# Deleting field 'Disk.ready'
db
.
delete_column
(
u'storage_disk'
,
'ready'
)
def
backwards
(
self
,
orm
):
# Adding field 'Disk.ready'
db
.
add_column
(
u'storage_disk'
,
'ready'
,
self
.
gf
(
'django.db.models.fields.BooleanField'
)(
default
=
False
),
keep_default
=
False
)
models
=
{
u'acl.level'
:
{
'Meta'
:
{
'unique_together'
:
"(('content_type', 'codename'),)"
,
'object_name'
:
'Level'
},
'codename'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
}),
'content_type'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['contenttypes.ContentType']"
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'50'
}),
'weight'
:
(
'django.db.models.fields.IntegerField'
,
[],
{
'null'
:
'True'
})
},
u'acl.objectlevel'
:
{
'Meta'
:
{
'unique_together'
:
"(('content_type', 'object_id', 'level'),)"
,
'object_name'
:
'ObjectLevel'
},
'content_type'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['contenttypes.ContentType']"
}),
'groups'
:
(
'django.db.models.fields.related.ManyToManyField'
,
[],
{
'to'
:
u"orm['auth.Group']"
,
'symmetrical'
:
'False'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'level'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['acl.Level']"
}),
'object_id'
:
(
'django.db.models.fields.IntegerField'
,
[],
{}),
'users'
:
(
'django.db.models.fields.related.ManyToManyField'
,
[],
{
'to'
:
u"orm['auth.User']"
,
'symmetrical'
:
'False'
})
},
u'auth.group'
:
{
'Meta'
:
{
'object_name'
:
'Group'
},
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'80'
}),
'permissions'
:
(
'django.db.models.fields.related.ManyToManyField'
,
[],
{
'to'
:
u"orm['auth.Permission']"
,
'symmetrical'
:
'False'
,
'blank'
:
'True'
})
},
u'auth.permission'
:
{
'Meta'
:
{
'ordering'
:
"(u'content_type__app_label', u'content_type__model', u'codename')"
,
'unique_together'
:
"((u'content_type', u'codename'),)"
,
'object_name'
:
'Permission'
},
'codename'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
}),
'content_type'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['contenttypes.ContentType']"
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'50'
})
},
u'auth.user'
:
{
'Meta'
:
{
'object_name'
:
'User'
},
'date_joined'
:
(
'django.db.models.fields.DateTimeField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'email'
:
(
'django.db.models.fields.EmailField'
,
[],
{
'max_length'
:
'75'
,
'blank'
:
'True'
}),
'first_name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'30'
,
'blank'
:
'True'
}),
'groups'
:
(
'django.db.models.fields.related.ManyToManyField'
,
[],
{
'to'
:
u"orm['auth.Group']"
,
'symmetrical'
:
'False'
,
'blank'
:
'True'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'is_active'
:
(
'django.db.models.fields.BooleanField'
,
[],
{
'default'
:
'True'
}),
'is_staff'
:
(
'django.db.models.fields.BooleanField'
,
[],
{
'default'
:
'False'
}),
'is_superuser'
:
(
'django.db.models.fields.BooleanField'
,
[],
{
'default'
:
'False'
}),
'last_login'
:
(
'django.db.models.fields.DateTimeField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'last_name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'30'
,
'blank'
:
'True'
}),
'password'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'128'
}),
'user_permissions'
:
(
'django.db.models.fields.related.ManyToManyField'
,
[],
{
'to'
:
u"orm['auth.Permission']"
,
'symmetrical'
:
'False'
,
'blank'
:
'True'
}),
'username'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'30'
})
},
u'contenttypes.contenttype'
:
{
'Meta'
:
{
'ordering'
:
"('name',)"
,
'unique_together'
:
"(('app_label', 'model'),)"
,
'object_name'
:
'ContentType'
,
'db_table'
:
"'django_content_type'"
},
'app_label'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'model'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
})
},
u'storage.datastore'
:
{
'Meta'
:
{
'ordering'
:
"['name']"
,
'object_name'
:
'DataStore'
},
'hostname'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'40'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'100'
}),
'path'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'200'
})
},
u'storage.disk'
:
{
'Meta'
:
{
'ordering'
:
"['name']"
,
'object_name'
:
'Disk'
},
'base'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'blank'
:
'True'
,
'related_name'
:
"'derivatives'"
,
'null'
:
'True'
,
'to'
:
u"orm['storage.Disk']"
}),
'created'
:
(
'model_utils.fields.AutoCreatedField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'datastore'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['storage.DataStore']"
}),
'destroyed'
:
(
'django.db.models.fields.DateTimeField'
,
[],
{
'default'
:
'None'
,
'null'
:
'True'
,
'blank'
:
'True'
}),
'dev_num'
:
(
'django.db.models.fields.CharField'
,
[],
{
'default'
:
"'a'"
,
'max_length'
:
'1'
}),
'filename'
:
(
'django.db.models.fields.CharField'
,
[],
{
'unique'
:
'True'
,
'max_length'
:
'256'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'modified'
:
(
'model_utils.fields.AutoLastModifiedField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'name'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
,
'blank'
:
'True'
}),
'size'
:
(
'sizefield.models.FileSizeField'
,
[],
{}),
'type'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'10'
})
},
u'storage.diskactivity'
:
{
'Meta'
:
{
'object_name'
:
'DiskActivity'
},
'activity_code'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'100'
}),
'created'
:
(
'model_utils.fields.AutoCreatedField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'disk'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'related_name'
:
"'activity_log'"
,
'to'
:
u"orm['storage.Disk']"
}),
'finished'
:
(
'django.db.models.fields.DateTimeField'
,
[],
{
'null'
:
'True'
,
'blank'
:
'True'
}),
u'id'
:
(
'django.db.models.fields.AutoField'
,
[],
{
'primary_key'
:
'True'
}),
'modified'
:
(
'model_utils.fields.AutoLastModifiedField'
,
[],
{
'default'
:
'datetime.datetime.now'
}),
'parent'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'blank'
:
'True'
,
'related_name'
:
"'children'"
,
'null'
:
'True'
,
'to'
:
u"orm['storage.DiskActivity']"
}),
'result'
:
(
'django.db.models.fields.TextField'
,
[],
{
'null'
:
'True'
,
'blank'
:
'True'
}),
'started'
:
(
'django.db.models.fields.DateTimeField'
,
[],
{
'null'
:
'True'
,
'blank'
:
'True'
}),
'succeeded'
:
(
'django.db.models.fields.NullBooleanField'
,
[],
{
'null'
:
'True'
,
'blank'
:
'True'
}),
'task_uuid'
:
(
'django.db.models.fields.CharField'
,
[],
{
'max_length'
:
'50'
,
'unique'
:
'True'
,
'null'
:
'True'
,
'blank'
:
'True'
}),
'user'
:
(
'django.db.models.fields.related.ForeignKey'
,
[],
{
'to'
:
u"orm['auth.User']"
,
'null'
:
'True'
,
'blank'
:
'True'
})
}
}
complete_apps
=
[
'storage'
]
\ No newline at end of file
circle/storage/models.py
View file @
e60ac87d
...
...
@@ -5,7 +5,7 @@ import logging
from
os.path
import
join
import
uuid
from
django.db.models
import
(
Model
,
BooleanField
,
CharField
,
DateTimeField
,
from
django.db.models
import
(
Model
,
CharField
,
DateTimeField
,
ForeignKey
)
from
django.utils
import
timezone
from
django.utils.translation
import
ugettext_lazy
as
_
...
...
@@ -17,7 +17,8 @@ from acl.models import AclBase
from
.tasks
import
local_tasks
,
remote_tasks
from
celery.exceptions
import
TimeoutError
from
manager.mancelery
import
celery
from
common.models
import
ActivityModel
,
activitycontextimpl
,
WorkerNotFound
from
common.models
import
(
ActivityModel
,
activitycontextimpl
,
WorkerNotFound
)
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -74,8 +75,6 @@ class Disk(AclBase, TimeStampedModel):
size
=
FileSizeField
()
base
=
ForeignKey
(
'self'
,
blank
=
True
,
null
=
True
,
related_name
=
'derivatives'
)
ready
=
BooleanField
(
default
=
False
,
help_text
=
_
(
"The associated resource is ready."
))
dev_num
=
CharField
(
default
=
'a'
,
max_length
=
1
,
verbose_name
=
_
(
"device number"
))
destroyed
=
DateTimeField
(
blank
=
True
,
default
=
None
,
null
=
True
)
...
...
@@ -109,6 +108,11 @@ class Disk(AclBase, TimeStampedModel):
self
.
disk
=
disk
@property
def
ready
(
self
):
return
self
.
activity_log
.
filter
(
activity_code__endswith
=
"deploy"
,
succeeded__isnull
=
False
)
@property
def
path
(
self
):
"""The path where the files are stored.
"""
...
...
@@ -151,15 +155,16 @@ class Disk(AclBase, TimeStampedModel):
}[
self
.
type
]
def
is_downloading
(
self
):
da
=
DiskActivity
.
objects
.
filter
(
disk
=
self
)
.
latest
(
"created"
)
return
(
da
.
activity_code
==
"storage.Disk.download"
and
da
.
succeeded
is
Non
e
)
return
self
.
activity_log
.
filter
(
activity_code__endswith
=
"downloading_disk"
,
succeeded__isnull
=
Tru
e
)
def
get_download_percentage
(
self
):
if
not
self
.
is_downloading
():
return
None
task
=
DiskActivity
.
objects
.
latest
(
"created"
)
.
task_uuid
task
=
self
.
activity_log
.
filter
(
activity_code__endswith
=
"deploy"
,
succeeded__isnull
=
True
)[
0
]
.
task_uuid
result
=
celery
.
AsyncResult
(
id
=
task
)
return
result
.
info
.
get
(
"percent"
)
...
...
@@ -268,8 +273,7 @@ class Disk(AclBase, TimeStampedModel):
self
.
save
()
if
self
.
ready
:
return
False
return
True
with
disk_activity
(
code_suffix
=
'deploy'
,
disk
=
self
,
task_uuid
=
task_uuid
,
user
=
user
)
as
act
:
...
...
@@ -287,9 +291,6 @@ class Disk(AclBase, TimeStampedModel):
queue
=
queue_name
)
.
get
(
timeout
=
timeout
)
self
.
ready
=
True
self
.
save
()
return
True
def
deploy_async
(
self
,
user
=
None
):
...
...
@@ -299,10 +300,17 @@ class Disk(AclBase, TimeStampedModel):
queue
=
"localhost.man"
)
@classmethod
def
create
(
cls
,
**
params
):
def
create
(
cls
,
instance
=
None
,
user
=
None
,
**
params
):
"""Create disk with activity.
"""
datastore
=
params
.
pop
(
'datastore'
,
DataStore
.
objects
.
get
())
disk
=
cls
(
filename
=
str
(
uuid
.
uuid4
()),
datastore
=
datastore
,
**
params
)
disk
.
save
()
with
disk_activity
(
code_suffix
=
"create"
,
user
=
user
,
disk
=
disk
):
if
instance
:
instance
.
disks
.
add
(
disk
)
return
disk
@classmethod
...
...
@@ -316,10 +324,7 @@ class Disk(AclBase, TimeStampedModel):
:return: Disk object without a real image, to be .deploy()ed later.
"""
disk
=
cls
.
create
(
**
kwargs
)
with
disk_activity
(
code_suffix
=
"create"
,
user
=
user
,
disk
=
disk
):
if
instance
:
instance
.
disks
.
add
(
disk
)
disk
=
Disk
.
create
(
instance
=
None
,
user
=
None
,
**
kwargs
)
return
disk
@classmethod
...
...
@@ -352,18 +357,17 @@ class Disk(AclBase, TimeStampedModel):
:type instance: vm.models.Instance or InstanceTemplate or NoneType
:param user: owner of the disk
:type user: django.contrib.auth.User
:param task_uuid:
TODO
:param abortable_task:
TODO
:param task_uuid:
UUID of the local task
:param abortable_task:
UUID of the remote running abortable task.
:return: The created Disk object
:rtype: Disk
"""
kwargs
.
setdefault
(
'name'
,
url
.
split
(
'/'
)[
-
1
])
disk
=
Disk
.
create
(
type
=
"iso"
,
size
=
1
,
**
kwargs
)
disk
=
Disk
.
create
(
type
=
"iso"
,
instance
=
instance
,
user
=
user
,
size
=
1
,
**
kwargs
)
# TODO get proper datastore
disk
.
datastore
=
DataStore
.
objects
.
get
()
if
instance
:
instance
.
disks
.
add
(
disk
)
queue_name
=
disk
.
get_remote_queue_name
(
'storage'
)
def
__on_abort
(
activity
,
error
):
...
...
@@ -376,9 +380,10 @@ class Disk(AclBase, TimeStampedModel):
class
AbortException
(
Exception
):
pass
with
disk_activity
(
code_suffix
=
'd
ownload
'
,
disk
=
disk
,
with
disk_activity
(
code_suffix
=
'd
eploy
'
,
disk
=
disk
,
task_uuid
=
task_uuid
,
user
=
user
,
on_abort
=
__on_abort
):
on_abort
=
__on_abort
)
as
act
:
with
act
.
sub_activity
(
'downloading_disk'
):
result
=
remote_tasks
.
download
.
apply_async
(
kwargs
=
{
'url'
:
url
,
'parent_id'
:
task_uuid
,
'disk'
:
disk
.
get_disk_desc
()},
...
...
@@ -392,7 +397,6 @@ class Disk(AclBase, TimeStampedModel):
AbortableAsyncResult
(
result
.
id
)
.
abort
()
raise
AbortException
(
"Download aborted by user."
)
disk
.
size
=
size
disk
.
ready
=
True
disk
.
save
()
return
disk
...
...
@@ -453,15 +457,14 @@ class Disk(AclBase, TimeStampedModel):
disk
.
save
()
with
disk_activity
(
code_suffix
=
"save_as"
,
disk
=
self
,
user
=
user
,
task_uuid
=
None
):
user
=
user
,
task_uuid
=
task_uuid
):
with
disk_activity
(
code_suffix
=
"deploy"
,
disk
=
disk
,
user
=
user
,
task_uuid
=
task_uuid
):
queue_name
=
self
.
get_remote_queue_name
(
'storage'
)
remote_tasks
.
merge
.
apply_async
(
args
=
[
self
.
get_disk_desc
(),
disk
.
get_disk_desc
()],
queue
=
queue_name
)
.
get
()
# Timeout
disk
.
ready
=
True
disk
.
save
()
return
disk
...
...
@@ -478,6 +481,15 @@ class DiskActivity(ActivityModel):
act
.
save
()
return
act
def
__unicode__
(
self
):
if
self
.
parent
:
return
'{}({})->{}'
.
format
(
self
.
parent
.
activity_code
,
self
.
disk
,
self
.
activity_code
)
else
:
return
'{}({})'
.
format
(
self
.
activity_code
,
self
.
disk
)
def
create_sub
(
self
,
code_suffix
,
task_uuid
=
None
):
act
=
DiskActivity
(
activity_code
=
self
.
activity_code
+
'.'
+
code_suffix
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment