Skip to content

Ludmány Balázs / thin-client

Go to a project
Commit 3aeb58bf by Ludmány Balázs
Options

Reuse buffers

parent 48d85ad7
Showing with 468 additions and 388 deletions
ThinClient.pro
TEMPLATE = app
QT += qml quick
CONFIG += c++11
CONFIG += c++14
SOURCES += main.cpp \
jpegdecoder.cpp \
......
ThinClient.pro.user
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.5.1, 2016-07-20T14:21:50. -->
<!-- Written by QtCreator 3.5.1, 2016-07-26T16:47:48. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>
......
copy_shader.fsh
uniform sampler2D texture;
varying highp vec2 vartexcoord;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb, 1.0);
//gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
}
copy_shader.vsh
uniform highp mat4 ortho;
attribute highp vec2 position;
attribute highp vec2 texcoord;
varying highp vec2 vartexcoord;
uniform mediump mat4 ortho;
uniform mediump mat4 texortho;
attribute mediump vec2 position;
attribute mediump vec2 texcoord;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_Position = vec4(position, 1.0, 1.0);
highp vec4 temp = ortho * vec4(texcoord, 0.0, 1.0);
vartexcoord = temp.xy * vec2(0.5, 0.5) + vec2(0.5, 0.5);
gl_Position = vec4((position.x - 1024.0) / 1024.0, (position.y - 1024.0) / 1024.0, 0.0, 1.0);
vartexcoord = vec2(0.0, 1.0) - vec2(texcoord.x / 1280.0, texcoord.y / 720.0);
}
dispatcher.cpp
......@@ -12,7 +12,7 @@ rfbBool Dispatcher::MallocFrameBuffer(rfbClient *client)
return TRUE;
}
void Dispatcher::GotCopyRect(rfbClient *client, int src_x, int src_y, int w, int h, int dest_x, int dest_y)
void Dispatcher::GotCopyRect(rfbClient *client, const int src_x, const int src_y, const int w, const int h, const int dest_x, const int dest_y)
{
Dispatcher *dispatcher = (Dispatcher *) rfbClientGetClientData(client, 0);
dispatcher->uploader()->gotCopy(src_x, src_y, w, h, dest_x, dest_y);
......@@ -31,13 +31,15 @@ void Dispatcher::GotBitmap(rfbClient *client, const uint8_t *buffer, int x, int
{
Dispatcher *dispatcher = (Dispatcher *) rfbClientGetClientData(client, 0);
// We have to make a deep copy because libvnc might free the buffer as soon as this function returns
dispatcher->uploader()->gotBitmap(QByteArray((const char *) buffer, w * h * (dispatcher->bitsPerPixel() / 8)), x, y, w, h, GL_RGBA, GL_UNSIGNED_BYTE);
QImage image(buffer, w, h, QImage::Format_RGBX8888);
dispatcher->uploader()->gotBitmap(image.convertToFormat(QImage::Format_RGB888), x, y, w, h);
}
rfbBool Dispatcher::GotJpeg(rfbClient *client, const uint8_t *buffer, int length, int x, int y, int w, int h)
{
Dispatcher *dispatcher = static_cast<Dispatcher *>(rfbClientGetClientData(client, 0));
Jpeg *jpeg = new Jpeg((unsigned char*)buffer, length, dispatcher->uploader()->gotJpeg(x, y, w, h));
dispatcher->uploader()->gotJpeg();
Jpeg *jpeg = new Jpeg({(unsigned char*)buffer, length, x, y, w, h});
dispatcher->queue()->enqueue(jpeg);
return TRUE;
}
......@@ -70,13 +72,6 @@ int Dispatcher::bitsPerPixel()
return m_client->format.bitsPerPixel;
}
QVector3D Dispatcher::colorMax() const
{
if(m_client == NULL)
return QVector3D();
return QVector3D(m_client->format.redMax, m_client->format.greenMax, m_client->format.blueMax);
}
ConcurrentQueue<Jpeg *, 256u> *Dispatcher::queue()
{
return &m_queue;
......@@ -107,8 +102,8 @@ void Dispatcher::open(const QString host, const int port, const int width, const
// Set the format requested by the user
m_client->width = width;
m_client->height = height;
m_client->appData.compressLevel = 0;
m_client->appData.qualityLevel = 0;
//m_client->appData.compressLevel = 0;
//m_client->appData.qualityLevel = 0;
#ifdef __BIG_ENDIAN__
m_client->format.bigEndian = TRUE;
#else
......@@ -124,7 +119,6 @@ void Dispatcher::open(const QString host, const int port, const int width, const
emit error();
return;
}
emit colorMaxChanged(colorMax());
// Stick to the format sent by the server if we can render it easily, set a new one otherwise
//
......@@ -156,25 +150,23 @@ void Dispatcher::refresh()
if(m_client == NULL)
return;
m_uploader->cleanup();
int message = 0;
if(!HandleRFBServerMessage(m_client)) {
// Wait until we get something
while(message == 0) {
message = WaitForMessage(m_client, 1000);
if(message < 0) {
terminate();
emit error();
return;
}
}
// Wait until we get something
while(message == 0) {
message = WaitForMessage(m_client, 10000);
if(message < 0) {
if(!HandleRFBServerMessage(m_client)) {
terminate();
emit error();
return;
}
}
}
void Dispatcher::terminate()
......
dispatcher.h
......@@ -25,7 +25,7 @@ public:
// libvnc hooks
static rfbBool MallocFrameBuffer(rfbClient* client);
static void GotCopyRect(rfbClient* client, int src_x, int src_y, int w, int h, int dest_x, int dest_y);
static void GotCopyRect(rfbClient* client, const int src_x, const int src_y, const int w, const int h, const int dest_x, const int dest_y);
static void GotFillRect(rfbClient* client, int x, int y, int w, int h, uint32_t colour);
static void GotBitmap(rfbClient* client, const uint8_t* buffer, int x, int y, int w, int h);
static rfbBool GotJpeg(rfbClient* client, const uint8_t* buffer, int length, int x, int y, int w, int h);
......@@ -36,7 +36,6 @@ public:
// parameters of the VNC connection
int bitsPerPixel();
QVector3D colorMax() const;
// getters
ConcurrentQueue<Jpeg *, 256u>* queue();
......@@ -52,7 +51,6 @@ signals:
void gotBitmap(const QImage &image, const quint32 x, const quint32 y, const quint32 width, const quint32 height);
void finishedUpdate();
void resizeFramebuffer(const QSize &size);
void colorMaxChanged(const QVector3D &colorMax);
void error();
public slots:
// manage VNC connection
......
draw_shader.fsh
uniform sampler2D texture;
varying lowp vec2 vartexcoord;
varying lowp vec3 varcolor;
varying mediump vec2 vartexcoord;
varying lowp vec4 varcolor;
void main(void)
{
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb + varcolor, 1.0);
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb, 1.0) + varcolor;
// gl_FragColor = vec4(vartexcoord, 0.0, 1.0);
}
draw_shader.vsh
uniform lowp mat4 ortho;
uniform lowp vec3 colormax;
attribute lowp vec2 position;
attribute lowp vec3 color;
attribute lowp vec2 texcoord;
varying lowp vec3 varcolor;
varying lowp vec2 vartexcoord;
uniform mediump mat4 ortho;
attribute mediump vec2 position;
attribute lowp vec4 color;
attribute mediump vec2 texcoord;
varying lowp vec4 varcolor;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_Position = ortho * vec4(position, 0.0, 1.0);
varcolor = color / colormax;
vartexcoord = texcoord;
varcolor = color.gggg;
vartexcoord = vec2(texcoord.x / 2048.0, texcoord.y / 2048.0);
}
jpegdecoder.cpp
......@@ -22,7 +22,7 @@ void JpegDecoder::operate()
}
jpeg_create_decompress(&cinfo);
cinfo.do_fancy_upsampling = TRUE;
cinfo.dct_method = JDCT_IFAST;
cinfo.dct_method = JDCT_FLOAT;
jpeg_mem_src(&cinfo, jpeg->data, jpeg->length);
(void) jpeg_read_header(&cinfo, TRUE);
......@@ -38,7 +38,8 @@ void JpegDecoder::operate()
jpeg_read_scanlines(&cinfo, rows + cinfo.output_scanline, 1);
}
(void) jpeg_finish_decompress(&cinfo);
emit finished(samples, cinfo.output_width, cinfo.output_height, jpeg->index);
emit finished(samples, jpeg->x, jpeg->y, jpeg->width, jpeg->height);
delete[] jpeg->data;
delete jpeg;
delete rows;
jpeg_destroy_decompress(&cinfo);
......
jpegdecoder.h
......@@ -14,18 +14,12 @@
struct Jpeg
{
Jpeg(uint8_t *d, const int l, const int i) :
data(d), length(l), index(i)
{
}
~Jpeg()
{
delete[] data;
}
uint8_t *data;
int length;
int index;
int x;
int y;
int width;
int height;
};
/*!
......@@ -40,7 +34,7 @@ public:
JpegDecoder(ConcurrentQueue<Jpeg*, 256u> *queue, QObject *parent = 0);
ConcurrentQueue<Jpeg*, 256u> *m_queue;
signals:
void finished(const unsigned char *image, const int width, const int height, const int index);
void finished(const unsigned char *image, const quint32 x, const quint32 y, const quint32 width, const quint32 height);
public slots:
void operate();
};
......
main.qml
......@@ -4,7 +4,8 @@ import thinclient 1.3
ApplicationWindow {
visible: true
visibility: "Maximized"
width: vnc.width
height: tab.height + vnc.height
id: window
header: TabBar {
......@@ -27,7 +28,7 @@ ApplicationWindow {
id: vnc
host: "vm.ik.bme.hu"
port: 10495
width: window.width
height: window.height - tab.height
width: 1280
height: 720
}
}
qvnc.cpp
......@@ -25,6 +25,7 @@ QVnc::QVnc(QQuickItem *parent) :
connect(this, &QVnc::open_connection, m_dispatcher, &Dispatcher::open);
connect(this, &QVnc::terminate_connection, m_dispatcher, &Dispatcher::terminate);
connect(m_dispatcher, &Dispatcher::error, this, &QVnc::dispatch_error);
connect(m_uploader, &Uploader::uploadFinished, m_dispatcher, &Dispatcher::refresh);
int i = 1;
foreach(JpegDecoder* d, m_decoders) {
......@@ -85,17 +86,19 @@ void QVnc::dispatch_error()
void QVnc::winChanged(QQuickWindow *window)
{
if(window != NULL)
if(window != NULL) {
connect(window, &QQuickWindow::openglContextCreated, m_uploader, &Uploader::createContext, Qt::BlockingQueuedConnection);
connect(window, &QQuickWindow::frameSwapped, m_uploader, &Uploader::frameSwapped);
}
}
QQuickFramebufferObject::Renderer *QVnc::createRenderer() const
{
VncRenderer *renderer = new VncRenderer(window(),
m_uploader->vertices(),
m_uploader->textures());
connect(m_dispatcher, &Dispatcher::colorMaxChanged, renderer, &VncRenderer::setColorMax);
connect(renderer, &VncRenderer::finishedRendering, m_dispatcher, &Dispatcher::refresh);
m_uploader->indices(),
m_uploader->texture(),
m_uploader->drawCount());
connect(renderer, &VncRenderer::FBOTextureChanged, m_uploader, &Uploader::changeFBOTexture);
return renderer;
}
......
qvnc.h
#ifndef VNC_H
#define VNC_H
#define DECODER_COUNT 16
#define DECODER_COUNT 8
#include <QQuickWindow>
#include <QObject>
......
uploader.cpp
#include "uploader.h"
#include <cstring>
Uploader::Uploader(QObject *parent) : QObject(parent), m_context(this), m_allocated(0), m_program(this), m_jpegs(0)
Uploader::Uploader(QObject *parent) :
QObject(parent), m_context(this), m_indices(QOpenGLBuffer::IndexBuffer),
m_uploadVertices(new QOpenGLBuffer*), m_renderVertices(new QOpenGLBuffer*),
m_uploadTexture(new GLuint*), m_renderTexture(new GLuint*), m_program(this), m_jpegs(0),
m_atlasX(0), m_atlasY(0), m_atlasRowHeight(0), m_atlasLastWidth(0), m_copyIndex(0)
{
// "applications must ensure that create() is only called on the main (GUI) thread"
m_surface.create();
m_textures = QSharedPointer<QVector<GLuint>>(new QVector<GLuint>);
// reserve space for 10 rectangles
m_data.reserve(280);
m_textures->reserve(10);
m_drawCount = new unsigned int(0);
}
Uploader::~Uploader()
......@@ -20,325 +19,320 @@ Uploader::~Uploader()
void Uploader::createContext(QOpenGLContext *context)
{
static const float position[] = {-1.0f, -1.0f,
-1.0f, 1.0f,
1.0f, -1.0f,
1.0f, 1.0f};
m_context.setShareContext(context);
m_context.create();
m_context.makeCurrent(&m_surface);
m_functions = m_context.functions();
m_functions->glEnable(GL_CULL_FACE);
m_program.addShaderFromSourceFile(QOpenGLShader::Vertex, "copy_shader.vsh");
m_program.addShaderFromSourceFile(QOpenGLShader::Fragment, "copy_shader.fsh");
m_program.bind();
m_functions->glActiveTexture(GL_TEXTURE0);
m_program.setUniformValue("texture", 0);
QMatrix4x4 ortho;
ortho.ortho(0, TEXTURE_WIDTH, 0, TEXTURE_HEIGHT, -1, 1);
// m_program.setUniformValue("ortho", ortho);
m_program.enableAttributeArray("position");
m_program.enableAttributeArray("texcoord");
m_program.setAttributeArray("position", GL_FLOAT, position, 2);
m_functions->glVertexAttribPointer(m_program.attributeLocation("position"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Copy),
(void*) 0);
m_functions->glVertexAttribPointer(m_program.attributeLocation("texcoord"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Copy),
(void*) (2 * sizeof(GLushort)));
m_indices.create();
m_indices.bind();
m_indices.allocate(ind.i, (VERTEX_BUFFER * 6 - 2) * sizeof(unsigned short));
m_copyBuffer.create();
m_copyBuffer.bind();
m_copyBuffer.allocate(COPY_BUFFER * sizeof(Copy));
m_copyPointer = (Copy*) m_copyBuffer.map(QOpenGLBuffer::WriteOnly);
*m_renderVertices = new QOpenGLBuffer;
(*m_renderVertices)->setUsagePattern(QOpenGLBuffer::DynamicDraw);
(*m_renderVertices)->create();
(*m_renderVertices)->bind();
(*m_renderVertices)->allocate(VERTEX_BUFFER * sizeof(Vertex));
*m_uploadVertices = new QOpenGLBuffer;
(*m_uploadVertices)->setUsagePattern(QOpenGLBuffer::DynamicDraw);
(*m_uploadVertices)->create();
(*m_uploadVertices)->bind();
(*m_uploadVertices)->allocate(VERTEX_BUFFER * sizeof(Vertex));
m_vertexPointer = (Vertex*) (*m_uploadVertices)->map(QOpenGLBuffer::WriteOnly);
m_vertexIndex = 0;
m_functions->glEnable(GL_TEXTURE_2D);
m_functions->glActiveTexture(GL_TEXTURE0);
m_functions->glGenTextures(2, m_texId);
for(size_t i = 0; i < 2; i++) {
m_functions->glBindTexture(GL_TEXTURE_2D, m_texId[i]);
m_functions->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, TEXTURE_WIDTH, TEXTURE_HEIGHT, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
}
m_vertices.create();
m_vertices.setUsagePattern(QOpenGLBuffer::DynamicDraw);
}
*m_uploadTexture = m_texId;
*m_renderTexture = m_texId + 1;
void Uploader::cleanup()
{
m_data.clear();
m_functions->glDeleteTextures(m_textures->size(), m_textures->constData());
m_textures->clear();
m_finished = false;
m_functions->glBindTexture(GL_TEXTURE_2D, **m_uploadTexture);
m_functions->glGenFramebuffers(1, &m_FBOId);
m_functions->glBindFramebuffer(GL_FRAMEBUFFER, m_FBOId);
m_functions->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, **m_uploadTexture, 0);
m_functions->glViewport(0, 0, TEXTURE_WIDTH, TEXTURE_HEIGHT);
m_functions->glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
}
void Uploader::gotFill(const int x, const int y, const int width, const int height, const float red, const float green, const float blue)
void Uploader::gotFill(const GLushort x, const GLushort y,
const GLushort width, const GLushort height,
const GLubyte red, const GLubyte green, const GLubyte blue)
{
m_textures->append(0);
// NOTE: -coordinates are transformed to normalized device coordinates in the vertex shader
// -coordinates and colors are extended to vec4 in the vertex shader
// -OpenGL ES 2.0 does not support integer values as input,
// thus we need to cast to float on the CPU side :(
if(m_vertexIndex + 4 < VERTEX_BUFFER) {
// bottom left
m_data.append((float) x);
m_data.append((float) (y + height));
m_data.append(red);
m_data.append(green);
m_data.append(blue);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({x, (GLushort) (y + height), red, green, blue, 0xFF, 0, 0});
// top left
m_data.append((float) x);
m_data.append((float) y);
m_data.append(red);
m_data.append(green);
m_data.append(blue);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({x, y, red, green, blue, 0xFF, 0, 0});
// bottom right
m_data.append((float) (x + width));
m_data.append((float) (y + height));
m_data.append(red);
m_data.append(green);
m_data.append(blue);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), (GLushort) (y + height), red, green, blue, 0xFF, 0, 0});
// top right
m_data.append((float) (x + width));
m_data.append((float) y);
m_data.append(red);
m_data.append(green);
m_data.append(blue);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), y, red, green, blue, 0xFF, 0, 0});
} else {
qDebug() << "Out of memory";
}
}
void Uploader::gotCopy(const int src_x, const int src_y, const int width, const int height, const int dest_x, const int dest_y)
void Uploader::gotCopy(const GLushort src_x, const GLushort src_y,
const GLushort width, const GLushort height,
const GLushort dest_x, const GLushort dest_y)
{
float texcoord[] = {(float) src_x, (float) (src_y + height),
(float) src_x, (float) src_y,
(float) (src_x + width), (float) (src_y + height),
(float) (src_x + width), (float) src_y};
QOpenGLFramebufferObject fbo(width, height);
fbo.bind();
m_functions->glViewport(0, 0, width, height);
m_program.setAttributeArray("texcoord", GL_FLOAT, texcoord, 2);
m_functions->glBindTexture(GL_TEXTURE_2D, m_FBOTexture);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_functions->glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
m_textures->append(fbo.takeTexture());
if(!refreshAtlas(width, height))
return;
// copy from source
if(m_copyIndex + 4 < COPY_BUFFER) {
m_copyPointer[m_copyIndex++] = Copy({m_atlasX, m_atlasY, src_x, (GLushort) (src_y + height)});
m_copyPointer[m_copyIndex++] = Copy({m_atlasX, (GLushort) (m_atlasY + height), src_x, src_y});
m_copyPointer[m_copyIndex++] = Copy({(GLushort) (m_atlasX + width), m_atlasY, (GLushort) (src_x + width), (GLushort) (src_y + height)});
m_copyPointer[m_copyIndex++] = Copy({(GLushort) (m_atlasX + width), (GLushort) (m_atlasY + height), (GLushort) (src_x + width), src_y});
} else {
qDebug() << "Out of memory";
}
// copy to destination
if(m_vertexIndex + 4 < VERTEX_BUFFER) {
// bottom left
m_data.append((float) dest_x);
m_data.append((float) (dest_y + height));
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({dest_x, (GLushort) (dest_y + height), 0, 0, 0, 0, m_atlasX, m_atlasY});
// top left
m_data.append((float) dest_x);
m_data.append((float) dest_y);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(1.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({dest_x, dest_y, 0, 0, 0, 0, m_atlasX, (GLushort) (m_atlasY + height)});
// bottom right
m_data.append((float) (dest_x + width));
m_data.append((float) (dest_y + height));
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(1.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (dest_x + width), (GLushort) (dest_y + height), 0, 0, 0, 0, (GLushort) (m_atlasX + width), m_atlasY});
// top right
m_data.append((float) (dest_x + width));
m_data.append((float) dest_y);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(1.0f);
m_data.append(1.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (dest_x + width), dest_y, 0, 0, 0, 0, (GLushort) (m_atlasX + width), (GLushort) (m_atlasY + height)});
} else {
qDebug() << "Out of memory";
}
}
void Uploader::gotBitmap(const QByteArray &bitmap, const quint32 x, const quint32 y,
const quint32 width, const quint32 height, const GLenum format, const GLenum type)
void Uploader::gotBitmap(const QImage &image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height)
{
quint32 real_width = qNextPowerOfTwo(width - 1);
quint32 real_height = qNextPowerOfTwo(height - 1);
GLuint id;
m_functions->glGenTextures(1, &id);
m_functions->glBindTexture(GL_TEXTURE_2D, id);
m_functions->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, real_width, real_height, 0, format, type, NULL);
m_functions->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, format, type, bitmap.constData());
m_textures->append(id);
float bottom = (float) height / (float) real_height;
float right = (float) width / (float) real_width;
if(!refreshAtlas(width, height))
return;
// bottom left
m_data.append((float) x);
m_data.append((float) (y + height));
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_functions->glTexSubImage2D(GL_TEXTURE_2D, 0, m_atlasX, m_atlasY, width, height, GL_RGB, GL_UNSIGNED_BYTE, image.constBits());
m_data.append(0.0f);
m_data.append(bottom);
if(m_vertexIndex + 4 < VERTEX_BUFFER) {
// bottom left
m_vertexPointer[m_vertexIndex++] = Vertex({x, (GLushort) (y + height), 0, 0, 0, 0, m_atlasX, (GLushort) (m_atlasY + height)});
// top left
m_data.append((float) x);
m_data.append((float) y);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({x, y, 0, 0, 0, 0, m_atlasX, m_atlasY});
// bottom right
m_data.append((float) (x + width));
m_data.append((float) (y + height));
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(right);
m_data.append(bottom);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), (GLushort) (y + height), 0, 0, 0, 0, (GLushort) (m_atlasX + width), (GLushort) (m_atlasY + height)});
// top right
m_data.append((float) (x + width));
m_data.append((float) y);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(right);
m_data.append(0.0f);
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), y, 0, 0, 0, 0, (GLushort) (m_atlasX + width), m_atlasY});
} else {
qDebug() << "Out of memory";
}
}
// Allocate storage for the image
int Uploader::gotJpeg(const quint32 x, const quint32 y, const quint32 width, const quint32 height)
void Uploader::gotJpeg()
{
m_jpegs++;
}
// "This function returns the nearest power of two greater than value."
// We need greater than or equal
quint32 real_width = qNextPowerOfTwo(width - 1);
quint32 real_height = qNextPowerOfTwo(height - 1);
GLuint id;
m_functions->glGenTextures(1, &id);
m_functions->glBindTexture(GL_TEXTURE_2D, id);
#ifdef GL_RGB8
m_functions->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8, real_width, real_height, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
#else
#ifdef GL_RGB8_OES
m_functions->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB8_OES, real_width, real_height, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
#else
m_functions->glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, real_width, real_height, 0, GL_RGB, GL_UNSIGNED_BYTE, NULL);
#endif
#endif
m_textures->append(id);
float bottom = (float) height / (float) real_height;
float right = (float) width / (float) real_width;
void Uploader::finishedJpeg(const unsigned char *image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height)
{
if(!refreshAtlas(width, height))
return;
// bottom left
m_data.append((float) x);
m_data.append((float) (y + height));
m_jpegs--;
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_functions->glTexSubImage2D(GL_TEXTURE_2D, 0, m_atlasX, m_atlasY, width, height, GL_RGB, GL_UNSIGNED_BYTE, image);
delete[] image;
m_data.append(0.0f);
m_data.append(bottom);
if(m_vertexIndex + 4 < VERTEX_BUFFER) {
// bottom left
m_vertexPointer[m_vertexIndex++] = Vertex({x, (GLushort) (y + height), 0, 0, 0, 0, m_atlasX, (GLushort) (m_atlasY + height)});
// top left
m_data.append((float) x);
m_data.append((float) y);
m_vertexPointer[m_vertexIndex++] = Vertex({x, y, 0, 0, 0, 0, m_atlasX, m_atlasY});
// bottom right
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), (GLushort) (y + height), 0, 0, 0, 0, (GLushort) (m_atlasX + width), (GLushort) (m_atlasY + height)});
// top right
m_vertexPointer[m_vertexIndex++] = Vertex({(GLushort) (x + width), y, 0, 0, 0, 0, (GLushort) (m_atlasX + width), m_atlasY});
} else {
qDebug() << "Out of memory";
}
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
if(m_jpegs == 0 && m_finished && m_swapped) {
startRendering();
}
}
m_data.append(0.0f);
m_data.append(0.0f);
// bottom right
m_data.append((float) (x + width));
m_data.append((float) (y + height));
// TODO: the rendering thread might swap buffers before the startRendering signal reaches it
void Uploader::frameSwapped()
{
m_swapped = true;
if(m_finished) {
copyRectangles();
if(m_jpegs == 0) {
startRendering();
}
}
}
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
void Uploader::finishedUpdate()
{
m_finished = true;
if(m_swapped) {
copyRectangles();
if(m_jpegs == 0)
startRendering();
}
}
m_data.append(right);
m_data.append(bottom);
// top right
m_data.append((float) (x + width));
m_data.append((float) y);
bool Uploader::refreshAtlas(const int width, const int height)
{
if(m_atlasX + m_atlasLastWidth + width <= TEXTURE_WIDTH) {
if(m_atlasY + height > TEXTURE_HEIGHT) {
qDebug() << "Out of memory";
return false;
}
m_data.append(0.0f);
m_data.append(0.0f);
m_data.append(0.0f);
m_atlasX += m_atlasLastWidth;
m_atlasLastWidth = width;
m_data.append(right);
m_data.append(0.0f);
if(height > m_atlasRowHeight)
m_atlasRowHeight = height;
} else {
if(m_atlasY + m_atlasRowHeight + height > TEXTURE_HEIGHT) {
qDebug() << "Out of memory";
return false;
}
return m_textures->size() - 1;
m_atlasX = 0;
m_atlasY += m_atlasRowHeight;
m_atlasRowHeight = height;
m_atlasLastWidth = width;
}
return true;
}
void Uploader::finishedJpeg(const unsigned char *image, const quint32 width, const quint32 height, const int index)
void Uploader::startRendering()
{
m_jpegs--;
m_functions->glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
m_functions->glBindTexture(GL_TEXTURE_2D, m_textures->at(index));
m_functions->glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE, image);
delete[] image;
if(m_jpegs == 0 && m_finished) {
m_functions->glBindTexture(GL_TEXTURE_2D, 0);
m_functions->glFinish();
(*m_uploadVertices)->unmap();
std::swap(*m_uploadVertices, *m_renderVertices);
(*m_uploadVertices)->bind();
m_vertexPointer = (Vertex*) (*m_uploadVertices)->map(QOpenGLBuffer::WriteOnly);
if(m_vertexIndex > 0)
*m_drawCount = (m_vertexIndex / 4) * 6 - 2;
else
*m_drawCount = 0;
m_vertexIndex = 0;
std::swap(*m_uploadTexture, *m_renderTexture);
m_atlasX = 0;
m_atlasY = 0;
m_atlasRowHeight = 0;
m_atlasLastWidth = 0;
m_swapped = false;
m_finished = false;
m_functions->glBindTexture(GL_TEXTURE_2D, **m_uploadTexture);
m_functions->glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, **m_uploadTexture, 0);
emit uploadFinished();
}
}
void Uploader::finishedUpdate()
void Uploader::copyRectangles()
{
unsigned int size = m_data.size() * sizeof(float);
unsigned int nextPowerOfTwo = qNextPowerOfTwo(size);
m_vertices.bind();
if(m_copyIndex > 0) {
// m_functions->glActiveTexture(GL_TEXTURE1);
m_functions->glBindTexture(GL_TEXTURE_2D, m_FBOTexture);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_copyBuffer.bind();
m_copyBuffer.unmap();
if(nextPowerOfTwo > m_allocated) {
m_vertices.allocate(nextPowerOfTwo);
m_allocated = nextPowerOfTwo;
}
m_indices.bind();
m_vertices.write(0, m_data.constData(), size);
m_functions->glDrawElements(GL_TRIANGLE_STRIP, (m_copyIndex / 4) * 6 - 2, GL_UNSIGNED_SHORT, (GLvoid*) 0);
m_functions->glFinish();
m_vertices.release();
m_copyPointer = (Copy*) m_copyBuffer.map(QOpenGLBuffer::WriteOnly);
if(m_jpegs == 0) {
m_functions->glBindTexture(GL_TEXTURE_2D, 0);
emit uploadFinished();
} else {
m_finished = true;
(*m_uploadVertices)->bind();
m_functions->glBindTexture(GL_TEXTURE_2D, **m_uploadTexture);
// m_functions->glActiveTexture(GL_TEXTURE0);
m_copyIndex = 0;
}
}
void Uploader::swapBuffers()
QOpenGLBuffer **Uploader::vertices() const
{
// This thread should finish uploading
// The other thread should finish rendering from the other thread
m_functions->glFinish();
return m_renderVertices;
}
GLuint **Uploader::texture() const
{
return m_renderTexture;
}
QSharedPointer<QVector<GLuint>> Uploader::textures() const
QOpenGLBuffer Uploader::indices() const
{
return m_textures;
return m_indices;
}
QOpenGLBuffer Uploader::vertices() const
unsigned int *Uploader::drawCount() const
{
return m_vertices;
return m_drawCount;
}
void Uploader::changeFBOTexture(const unsigned int FBOTexture, const QMatrix4x4 &ortho)
{
/*m_functions->glActiveTexture(GL_TEXTURE1);
m_functions->glBindTexture(GL_TEXTURE_2D, m_FBOTexture);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_functions->glActiveTexture(GL_TEXTURE0);*/
m_FBOTexture = FBOTexture;
m_program.setUniformValue("ortho", ortho);
m_program.setUniformValue("texortho", ortho);
}
uploader.h
/*
* Design decisions:
* - There is no way to access texture memory directly, copying is required.
* See: https://www.raspberrypi.org/forums/viewtopic.php?f=67&t=25959
* - libvnc hooks free the memory buffer after they return (except for gotJpeg)
* Solutions:
* 1) copy images to separate textures
* Pro: simple
* Con: multiple textures (slow)
* 2) make deep copies of the images and move them to a single texture when decoding finished
* Pro: optimal texture atlas
* Con: deep copy
* 3) first-fit allocation
*/
#ifndef UPLOADER_H
#define UPLOADER_H
#define TEXTURE_WIDTH 2048
#define TEXTURE_HEIGHT 2048
#define COPY_BUFFER 1024
#define VERTEX_BUFFER 1024
#include <utility>
#include <QObject>
#include <QQuickWindow>
#include <QDebug>
......@@ -9,24 +31,53 @@
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QOpenGLBuffer>
#include <QOpenGLTexture>
#include <QOpenGLPixelTransferOptions>
#include <QOpenGLShaderProgram>
#include <QOpenGLFramebufferObject>
#include <QThreadPool>
#include <QtAlgorithms>
#include <QSharedPointer>
#include <QtMath>
class Bitmap {
public:
QByteArray data;
int x;
int y;
int width;
int height;
struct Indices
{
// 4 corner/rectangle and the first and last one duplicated except for the first and last rectangle
unsigned short i[(VERTEX_BUFFER / 4) * 6 - 2];
constexpr Indices() : i()
{
i[0] = 0;
auto index = 1;
for(auto value = 1; value < VERTEX_BUFFER; value++) {
i[index++] = value;
if(value % 4 == 0 || value % 4 == 3)
i[index++] = value;
}
}
};
constexpr Indices ind;
struct Vertex
{
GLushort x;
GLushort y;
GLubyte red;
GLubyte green;
GLubyte blue;
GLubyte alpha;
GLushort texX;
GLushort texY;
};
struct Copy
{
GLushort x;
GLushort y;
GLushort texX;
GLushort texY;
};
/*!
* \brief The Uploader class
*
* A framebuffer is bound to the
*/
class Uploader : public QObject
{
Q_OBJECT
......@@ -34,42 +85,83 @@ public:
Uploader(QObject *parent = Q_NULLPTR);
~Uploader();
QOpenGLBuffer vertices() const;
QSharedPointer<QVector<GLuint>> textures() const;
QOpenGLBuffer **vertices() const;
QOpenGLBuffer indices() const;
GLuint **texture() const;
unsigned int *drawCount() const;
// VNC events
void gotCopy(const int src_x, const int src_y, const int width, const int height, const int dest_x, const int dest_y);
void gotFill(const int x, const int y, const int width, const int height, const float red, const float green, const float blue);
int gotJpeg(const quint32 x, const quint32 y, const quint32 width, const quint32 height);
void gotBitmap(const QByteArray &bitmap, const quint32 x, const quint32 y, const quint32 width, const quint32 height, const GLenum format, const GLenum type);
void gotFill(const GLushort x, const GLushort y,
const GLushort width, const GLushort height,
const GLubyte red, const GLubyte green, const GLubyte blue);
void gotCopy(const GLushort src_x, const GLushort src_y,
const GLushort width, const GLushort height,
const GLushort dest_x, const GLushort dest_y);
void gotBitmap(const QImage &image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height);
void gotJpeg();
void finishedUpdate();
private:
QOffscreenSurface m_surface;
QOpenGLContext m_context;
QOpenGLFunctions *m_functions;
QOpenGLBuffer m_vertices;
unsigned int m_allocated;
QSharedPointer<QVector<GLuint>> m_textures;
QOpenGLBuffer m_indices;
QOpenGLBuffer **m_uploadVertices;
QOpenGLBuffer **m_renderVertices;
Vertex *m_vertexPointer;
size_t m_vertexIndex;
GLuint m_FBOId;
GLuint m_texId[2];
GLuint **m_uploadTexture;
GLuint **m_renderTexture;
unsigned int *m_drawCount;
QOpenGLShaderProgram m_program;
GLuint m_FBOTexture;
QVector<float> m_data;
QVector<Bitmap *> m_bitmaps;
int m_jpegs;
// Got a FinishedUpdate event from libvnc
bool m_finished;
// Framebuffer swapped
bool m_swapped;
void swapBuffers();
unsigned short m_atlasX;
unsigned short m_atlasY;
unsigned short m_atlasRowHeight;
unsigned short m_atlasLastWidth;
QOpenGLBuffer m_copyBuffer;
Copy *m_copyPointer;
size_t m_copyIndex;
bool refreshAtlas(const int width, const int height);
// Called when:
// - got a Finished update event from libvnc
// - and every jpeg was uploaded
// - and the rendering thread swapped buffers
void startRendering();
// Called when:
// - got a Finished update event from libvnc
// - and the rendering thread swapped buffers
void copyRectangles();
signals:
// Emit from startRendering only!
void uploadFinished();
public slots:
void changeFBOTexture(const unsigned int FBOTexture, const QMatrix4x4 &ortho);
void createContext(QOpenGLContext *context);
void cleanup();
void finishedJpeg(const unsigned char *image, const quint32 width, const quint32 height, int index);
void finishedJpeg(const unsigned char *image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height);
void frameSwapped();
};
#endif // UPLOADER_H
vncrenderer.cpp
#include "vncrenderer.h"
#include <cstring>
#include "uploader.h"
VncRenderer::VncRenderer(QQuickWindow *window,
QOpenGLBuffer vertices,
QSharedPointer<QVector<GLuint>> textures,
QOpenGLBuffer **vertices,
QOpenGLBuffer indices,
GLuint **texture,
unsigned int *drawCount,
QObject *parent) :
QObject(parent), m_window(window), m_vertices(vertices), m_textures(textures), m_rendered(false)
QObject(parent), m_window(window), m_vertices(vertices), m_indices(indices), m_texture(texture), m_drawCount(drawCount), m_rendered(false)
{
connect(m_window, &QQuickWindow::frameSwapped, this, &VncRenderer::frameSwapped);
m_program.addShaderFromSourceFile(QOpenGLShader::Vertex, "draw_shader.vsh");
m_program.addShaderFromSourceFile(QOpenGLShader::Fragment, "draw_shader.fsh");
m_program.bind();
m_program.setUniformValue("texture", 0);
QMatrix4x4 ortho;
ortho.ortho(QRect(QPoint(0, 0), QSize(2048, 2048)));
m_program.setUniformValue("texortho", ortho);
m_program.release();
m_functions = QOpenGLContext::currentContext()->functions();
......@@ -21,36 +26,50 @@ VncRenderer::VncRenderer(QQuickWindow *window,
void VncRenderer::render()
{
/*static QTime frameTime;
qDebug() << qRound(1000.0 / frameTime.elapsed()) << m_textures->length();
qDebug() << qRound(1000.0 / frameTime.elapsed()) << *m_drawCount;
frameTime.restart();*/
if(*m_drawCount > 0) {
m_program.bind();
m_vertices.bind();
m_program.setAttributeBuffer("position", GL_FLOAT, 0 * sizeof(float), 2, 7 * sizeof(float));
m_program.setAttributeBuffer("color", GL_FLOAT, 2 * sizeof(float), 3, 7 * sizeof(float));
m_program.setAttributeBuffer("texcoord", GL_FLOAT, 5 * sizeof(float), 2, 7 * sizeof(float));
// m_functions->glActiveTexture(GL_TEXTURE0);
(*m_vertices)->bind();
m_indices.bind();
m_functions->glVertexAttribPointer(m_program.attributeLocation("position"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Vertex),
(void*) 0);
m_functions->glVertexAttribPointer(m_program.attributeLocation("color"),
4,
GL_UNSIGNED_BYTE,
GL_TRUE,
sizeof(Vertex),
(void*) (2 * sizeof(GLushort)));
m_functions->glVertexAttribPointer(m_program.attributeLocation("texcoord"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Vertex),
(void*) (2 * sizeof(GLushort) + 4 * sizeof(GLubyte)));
m_program.enableAttributeArray("position");
m_program.enableAttributeArray("color");
m_program.enableAttributeArray("texcoord");
for(int i = 0; i < m_textures->length(); i++) {
m_functions->glBindTexture(GL_TEXTURE_2D, m_textures->at(i));
if(m_textures->at(i) != 0) {
m_functions->glBindTexture(GL_TEXTURE_2D, **m_texture);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
m_functions->glDrawArrays(GL_TRIANGLE_STRIP, i * 4, 4);
}
m_functions->glDrawElements(GL_TRIANGLE_STRIP, *m_drawCount, GL_UNSIGNED_SHORT, (void*) 0);
*m_drawCount = 0;
m_window->resetOpenGLState();
m_rendered = true;
}
}
QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size)
{
qDebug() << size;
QMatrix4x4 ortho;
ortho.ortho(QRect(QPoint(0, size.height()), QSize(size.width(), -1 * size.height())));
m_program.bind();
......@@ -62,8 +81,8 @@ QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size
#ifdef GL_RGBA8
format.setInternalTextureFormat(GL_RGBA8);
#else
#ifdef GL_RGBA8_OES
format.setInternalTextureFormat(GL_RGBA8_OES);
#ifdef RGBA8_OES
format.setInternalTextureFormat(RGBA8_OES);
#else
format.setInternalTextureFormat(GL_RGBA);
#endif
......@@ -73,19 +92,3 @@ QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size
emit FBOTextureChanged(framebufferObject->texture(), ortho);
return framebufferObject;
}
void VncRenderer::setColorMax(const QVector3D &colorMax)
{
if(m_program.bind()) {
m_program.setUniformValue("colormax", colorMax);
m_program.release();
}
}
void VncRenderer::frameSwapped()
{
if(m_rendered) {
m_rendered = false;
emit finishedRendering();
}
}
vncrenderer.h
......@@ -9,7 +9,6 @@
#include <QMatrix4x4>
#include <QOpenGLShaderProgram>
#include <QOpenGLBuffer>
#include <QSharedPointer>
#include <QTime>
class VncRenderer : public QObject, public QQuickFramebufferObject::Renderer
......@@ -17,24 +16,24 @@ class VncRenderer : public QObject, public QQuickFramebufferObject::Renderer
Q_OBJECT
public:
VncRenderer(QQuickWindow *window,
QOpenGLBuffer vertices,
QSharedPointer<QVector<GLuint>> textures,
QOpenGLBuffer **vertices,
QOpenGLBuffer indices,
GLuint **textures,
unsigned int *drawCount,
QObject *parent = 0);
protected:
void render() override;
QOpenGLFramebufferObject *createFramebufferObject(const QSize &size) override;
private:
QQuickWindow *m_window;
QOpenGLBuffer m_vertices;
QSharedPointer<QVector<GLuint>> m_textures;
QOpenGLBuffer **m_vertices;
QOpenGLBuffer m_indices;
GLuint **m_texture;
unsigned int *m_drawCount;
QVector3D m_colorMax;
QOpenGLShaderProgram m_program;
QOpenGLFunctions *m_functions;
bool m_rendered;
public slots:
void setColorMax(const QVector3D &colorMax);
private slots:
void frameSwapped();
signals:
void FBOTextureChanged(const unsigned int FBOTexture, const QMatrix4x4 &ortho);
void finishedRendering();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment