Commit 3aeb58bf by Ludmány Balázs

Reuse buffers

parent 48d85ad7
TEMPLATE = app
QT += qml quick
CONFIG += c++11
CONFIG += c++14
SOURCES += main.cpp \
jpegdecoder.cpp \
......
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE QtCreatorProject>
<!-- Written by QtCreator 3.5.1, 2016-07-20T14:21:50. -->
<!-- Written by QtCreator 3.5.1, 2016-07-26T16:47:48. -->
<qtcreator>
<data>
<variable>EnvironmentId</variable>
......
uniform sampler2D texture;
varying highp vec2 vartexcoord;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb, 1.0);
//gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
}
uniform highp mat4 ortho;
attribute highp vec2 position;
attribute highp vec2 texcoord;
varying highp vec2 vartexcoord;
uniform mediump mat4 ortho;
uniform mediump mat4 texortho;
attribute mediump vec2 position;
attribute mediump vec2 texcoord;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_Position = vec4(position, 1.0, 1.0);
highp vec4 temp = ortho * vec4(texcoord, 0.0, 1.0);
vartexcoord = temp.xy * vec2(0.5, 0.5) + vec2(0.5, 0.5);
gl_Position = vec4((position.x - 1024.0) / 1024.0, (position.y - 1024.0) / 1024.0, 0.0, 1.0);
vartexcoord = vec2(0.0, 1.0) - vec2(texcoord.x / 1280.0, texcoord.y / 720.0);
}
......@@ -12,7 +12,7 @@ rfbBool Dispatcher::MallocFrameBuffer(rfbClient *client)
return TRUE;
}
void Dispatcher::GotCopyRect(rfbClient *client, int src_x, int src_y, int w, int h, int dest_x, int dest_y)
void Dispatcher::GotCopyRect(rfbClient *client, const int src_x, const int src_y, const int w, const int h, const int dest_x, const int dest_y)
{
Dispatcher *dispatcher = (Dispatcher *) rfbClientGetClientData(client, 0);
dispatcher->uploader()->gotCopy(src_x, src_y, w, h, dest_x, dest_y);
......@@ -31,13 +31,15 @@ void Dispatcher::GotBitmap(rfbClient *client, const uint8_t *buffer, int x, int
{
Dispatcher *dispatcher = (Dispatcher *) rfbClientGetClientData(client, 0);
// We have to make a deep copy because libvnc might free the buffer as soon as this function returns
dispatcher->uploader()->gotBitmap(QByteArray((const char *) buffer, w * h * (dispatcher->bitsPerPixel() / 8)), x, y, w, h, GL_RGBA, GL_UNSIGNED_BYTE);
QImage image(buffer, w, h, QImage::Format_RGBX8888);
dispatcher->uploader()->gotBitmap(image.convertToFormat(QImage::Format_RGB888), x, y, w, h);
}
rfbBool Dispatcher::GotJpeg(rfbClient *client, const uint8_t *buffer, int length, int x, int y, int w, int h)
{
Dispatcher *dispatcher = static_cast<Dispatcher *>(rfbClientGetClientData(client, 0));
Jpeg *jpeg = new Jpeg((unsigned char*)buffer, length, dispatcher->uploader()->gotJpeg(x, y, w, h));
dispatcher->uploader()->gotJpeg();
Jpeg *jpeg = new Jpeg({(unsigned char*)buffer, length, x, y, w, h});
dispatcher->queue()->enqueue(jpeg);
return TRUE;
}
......@@ -70,13 +72,6 @@ int Dispatcher::bitsPerPixel()
return m_client->format.bitsPerPixel;
}
QVector3D Dispatcher::colorMax() const
{
if(m_client == NULL)
return QVector3D();
return QVector3D(m_client->format.redMax, m_client->format.greenMax, m_client->format.blueMax);
}
ConcurrentQueue<Jpeg *, 256u> *Dispatcher::queue()
{
return &m_queue;
......@@ -107,8 +102,8 @@ void Dispatcher::open(const QString host, const int port, const int width, const
// Set the format requested by the user
m_client->width = width;
m_client->height = height;
m_client->appData.compressLevel = 0;
m_client->appData.qualityLevel = 0;
//m_client->appData.compressLevel = 0;
//m_client->appData.qualityLevel = 0;
#ifdef __BIG_ENDIAN__
m_client->format.bigEndian = TRUE;
#else
......@@ -124,7 +119,6 @@ void Dispatcher::open(const QString host, const int port, const int width, const
emit error();
return;
}
emit colorMaxChanged(colorMax());
// Stick to the format sent by the server if we can render it easily, set a new one otherwise
//
......@@ -156,25 +150,23 @@ void Dispatcher::refresh()
if(m_client == NULL)
return;
m_uploader->cleanup();
int message = 0;
if(!HandleRFBServerMessage(m_client)) {
terminate();
emit error();
return;
}
// Wait until we get something
while(message == 0) {
message = WaitForMessage(m_client, 10000);
message = WaitForMessage(m_client, 1000);
if(message < 0) {
terminate();
emit error();
return;
}
}
if(!HandleRFBServerMessage(m_client)) {
terminate();
emit error();
return;
}
}
void Dispatcher::terminate()
......
......@@ -25,7 +25,7 @@ public:
// libvnc hooks
static rfbBool MallocFrameBuffer(rfbClient* client);
static void GotCopyRect(rfbClient* client, int src_x, int src_y, int w, int h, int dest_x, int dest_y);
static void GotCopyRect(rfbClient* client, const int src_x, const int src_y, const int w, const int h, const int dest_x, const int dest_y);
static void GotFillRect(rfbClient* client, int x, int y, int w, int h, uint32_t colour);
static void GotBitmap(rfbClient* client, const uint8_t* buffer, int x, int y, int w, int h);
static rfbBool GotJpeg(rfbClient* client, const uint8_t* buffer, int length, int x, int y, int w, int h);
......@@ -36,7 +36,6 @@ public:
// parameters of the VNC connection
int bitsPerPixel();
QVector3D colorMax() const;
// getters
ConcurrentQueue<Jpeg *, 256u>* queue();
......@@ -52,7 +51,6 @@ signals:
void gotBitmap(const QImage &image, const quint32 x, const quint32 y, const quint32 width, const quint32 height);
void finishedUpdate();
void resizeFramebuffer(const QSize &size);
void colorMaxChanged(const QVector3D &colorMax);
void error();
public slots:
// manage VNC connection
......
uniform sampler2D texture;
varying lowp vec2 vartexcoord;
varying lowp vec3 varcolor;
varying mediump vec2 vartexcoord;
varying lowp vec4 varcolor;
void main(void)
{
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb + varcolor, 1.0);
gl_FragColor = vec4(texture2D(texture, vartexcoord).rgb, 1.0) + varcolor;
// gl_FragColor = vec4(vartexcoord, 0.0, 1.0);
}
uniform lowp mat4 ortho;
uniform lowp vec3 colormax;
attribute lowp vec2 position;
attribute lowp vec3 color;
attribute lowp vec2 texcoord;
varying lowp vec3 varcolor;
varying lowp vec2 vartexcoord;
uniform mediump mat4 ortho;
attribute mediump vec2 position;
attribute lowp vec4 color;
attribute mediump vec2 texcoord;
varying lowp vec4 varcolor;
varying mediump vec2 vartexcoord;
void main(void)
{
gl_Position = ortho * vec4(position, 0.0, 1.0);
varcolor = color / colormax;
vartexcoord = texcoord;
varcolor = color.gggg;
vartexcoord = vec2(texcoord.x / 2048.0, texcoord.y / 2048.0);
}
......@@ -22,7 +22,7 @@ void JpegDecoder::operate()
}
jpeg_create_decompress(&cinfo);
cinfo.do_fancy_upsampling = TRUE;
cinfo.dct_method = JDCT_IFAST;
cinfo.dct_method = JDCT_FLOAT;
jpeg_mem_src(&cinfo, jpeg->data, jpeg->length);
(void) jpeg_read_header(&cinfo, TRUE);
......@@ -38,7 +38,8 @@ void JpegDecoder::operate()
jpeg_read_scanlines(&cinfo, rows + cinfo.output_scanline, 1);
}
(void) jpeg_finish_decompress(&cinfo);
emit finished(samples, cinfo.output_width, cinfo.output_height, jpeg->index);
emit finished(samples, jpeg->x, jpeg->y, jpeg->width, jpeg->height);
delete[] jpeg->data;
delete jpeg;
delete rows;
jpeg_destroy_decompress(&cinfo);
......
......@@ -14,18 +14,12 @@
struct Jpeg
{
Jpeg(uint8_t *d, const int l, const int i) :
data(d), length(l), index(i)
{
}
~Jpeg()
{
delete[] data;
}
uint8_t *data;
int length;
int index;
int x;
int y;
int width;
int height;
};
/*!
......@@ -40,7 +34,7 @@ public:
JpegDecoder(ConcurrentQueue<Jpeg*, 256u> *queue, QObject *parent = 0);
ConcurrentQueue<Jpeg*, 256u> *m_queue;
signals:
void finished(const unsigned char *image, const int width, const int height, const int index);
void finished(const unsigned char *image, const quint32 x, const quint32 y, const quint32 width, const quint32 height);
public slots:
void operate();
};
......
......@@ -4,7 +4,8 @@ import thinclient 1.3
ApplicationWindow {
visible: true
visibility: "Maximized"
width: vnc.width
height: tab.height + vnc.height
id: window
header: TabBar {
......@@ -27,7 +28,7 @@ ApplicationWindow {
id: vnc
host: "vm.ik.bme.hu"
port: 10495
width: window.width
height: window.height - tab.height
width: 1280
height: 720
}
}
......@@ -25,6 +25,7 @@ QVnc::QVnc(QQuickItem *parent) :
connect(this, &QVnc::open_connection, m_dispatcher, &Dispatcher::open);
connect(this, &QVnc::terminate_connection, m_dispatcher, &Dispatcher::terminate);
connect(m_dispatcher, &Dispatcher::error, this, &QVnc::dispatch_error);
connect(m_uploader, &Uploader::uploadFinished, m_dispatcher, &Dispatcher::refresh);
int i = 1;
foreach(JpegDecoder* d, m_decoders) {
......@@ -85,17 +86,19 @@ void QVnc::dispatch_error()
void QVnc::winChanged(QQuickWindow *window)
{
if(window != NULL)
if(window != NULL) {
connect(window, &QQuickWindow::openglContextCreated, m_uploader, &Uploader::createContext, Qt::BlockingQueuedConnection);
connect(window, &QQuickWindow::frameSwapped, m_uploader, &Uploader::frameSwapped);
}
}
QQuickFramebufferObject::Renderer *QVnc::createRenderer() const
{
VncRenderer *renderer = new VncRenderer(window(),
m_uploader->vertices(),
m_uploader->textures());
connect(m_dispatcher, &Dispatcher::colorMaxChanged, renderer, &VncRenderer::setColorMax);
connect(renderer, &VncRenderer::finishedRendering, m_dispatcher, &Dispatcher::refresh);
m_uploader->indices(),
m_uploader->texture(),
m_uploader->drawCount());
connect(renderer, &VncRenderer::FBOTextureChanged, m_uploader, &Uploader::changeFBOTexture);
return renderer;
}
......
#ifndef VNC_H
#define VNC_H
#define DECODER_COUNT 16
#define DECODER_COUNT 8
#include <QQuickWindow>
#include <QObject>
......
/*
* Design decisions:
* - There is no way to access texture memory directly, copying is required.
* See: https://www.raspberrypi.org/forums/viewtopic.php?f=67&t=25959
* - libvnc hooks free the memory buffer after they return (except for gotJpeg)
* Solutions:
* 1) copy images to separate textures
* Pro: simple
* Con: multiple textures (slow)
* 2) make deep copies of the images and move them to a single texture when decoding finished
* Pro: optimal texture atlas
* Con: deep copy
* 3) first-fit allocation
*/
#ifndef UPLOADER_H
#define UPLOADER_H
#define TEXTURE_WIDTH 2048
#define TEXTURE_HEIGHT 2048
#define COPY_BUFFER 1024
#define VERTEX_BUFFER 1024
#include <utility>
#include <QObject>
#include <QQuickWindow>
#include <QDebug>
......@@ -9,24 +31,53 @@
#include <QOpenGLContext>
#include <QOpenGLFunctions>
#include <QOpenGLBuffer>
#include <QOpenGLTexture>
#include <QOpenGLPixelTransferOptions>
#include <QOpenGLShaderProgram>
#include <QOpenGLFramebufferObject>
#include <QThreadPool>
#include <QtAlgorithms>
#include <QSharedPointer>
#include <QtMath>
class Bitmap {
public:
QByteArray data;
int x;
int y;
int width;
int height;
struct Indices
{
// 4 corner/rectangle and the first and last one duplicated except for the first and last rectangle
unsigned short i[(VERTEX_BUFFER / 4) * 6 - 2];
constexpr Indices() : i()
{
i[0] = 0;
auto index = 1;
for(auto value = 1; value < VERTEX_BUFFER; value++) {
i[index++] = value;
if(value % 4 == 0 || value % 4 == 3)
i[index++] = value;
}
}
};
constexpr Indices ind;
struct Vertex
{
GLushort x;
GLushort y;
GLubyte red;
GLubyte green;
GLubyte blue;
GLubyte alpha;
GLushort texX;
GLushort texY;
};
struct Copy
{
GLushort x;
GLushort y;
GLushort texX;
GLushort texY;
};
/*!
* \brief The Uploader class
*
* A framebuffer is bound to the
*/
class Uploader : public QObject
{
Q_OBJECT
......@@ -34,42 +85,83 @@ public:
Uploader(QObject *parent = Q_NULLPTR);
~Uploader();
QOpenGLBuffer vertices() const;
QSharedPointer<QVector<GLuint>> textures() const;
QOpenGLBuffer **vertices() const;
QOpenGLBuffer indices() const;
GLuint **texture() const;
unsigned int *drawCount() const;
// VNC events
void gotCopy(const int src_x, const int src_y, const int width, const int height, const int dest_x, const int dest_y);
void gotFill(const int x, const int y, const int width, const int height, const float red, const float green, const float blue);
int gotJpeg(const quint32 x, const quint32 y, const quint32 width, const quint32 height);
void gotBitmap(const QByteArray &bitmap, const quint32 x, const quint32 y, const quint32 width, const quint32 height, const GLenum format, const GLenum type);
void gotFill(const GLushort x, const GLushort y,
const GLushort width, const GLushort height,
const GLubyte red, const GLubyte green, const GLubyte blue);
void gotCopy(const GLushort src_x, const GLushort src_y,
const GLushort width, const GLushort height,
const GLushort dest_x, const GLushort dest_y);
void gotBitmap(const QImage &image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height);
void gotJpeg();
void finishedUpdate();
private:
QOffscreenSurface m_surface;
QOpenGLContext m_context;
QOpenGLFunctions *m_functions;
QOpenGLBuffer m_vertices;
unsigned int m_allocated;
QSharedPointer<QVector<GLuint>> m_textures;
QOpenGLBuffer m_indices;
QOpenGLBuffer **m_uploadVertices;
QOpenGLBuffer **m_renderVertices;
Vertex *m_vertexPointer;
size_t m_vertexIndex;
GLuint m_FBOId;
GLuint m_texId[2];
GLuint **m_uploadTexture;
GLuint **m_renderTexture;
unsigned int *m_drawCount;
QOpenGLShaderProgram m_program;
GLuint m_FBOTexture;
QVector<float> m_data;
QVector<Bitmap *> m_bitmaps;
int m_jpegs;
// Got a FinishedUpdate event from libvnc
bool m_finished;
// Framebuffer swapped
bool m_swapped;
void swapBuffers();
unsigned short m_atlasX;
unsigned short m_atlasY;
unsigned short m_atlasRowHeight;
unsigned short m_atlasLastWidth;
QOpenGLBuffer m_copyBuffer;
Copy *m_copyPointer;
size_t m_copyIndex;
bool refreshAtlas(const int width, const int height);
// Called when:
// - got a Finished update event from libvnc
// - and every jpeg was uploaded
// - and the rendering thread swapped buffers
void startRendering();
// Called when:
// - got a Finished update event from libvnc
// - and the rendering thread swapped buffers
void copyRectangles();
signals:
// Emit from startRendering only!
void uploadFinished();
public slots:
void changeFBOTexture(const unsigned int FBOTexture, const QMatrix4x4 &ortho);
void createContext(QOpenGLContext *context);
void cleanup();
void finishedJpeg(const unsigned char *image, const quint32 width, const quint32 height, int index);
void finishedJpeg(const unsigned char *image,
const GLushort x, const GLushort y,
const GLushort width, const GLushort height);
void frameSwapped();
};
#endif // UPLOADER_H
#include "vncrenderer.h"
#include <cstring>
#include "uploader.h"
VncRenderer::VncRenderer(QQuickWindow *window,
QOpenGLBuffer vertices,
QSharedPointer<QVector<GLuint>> textures,
QOpenGLBuffer **vertices,
QOpenGLBuffer indices,
GLuint **texture,
unsigned int *drawCount,
QObject *parent) :
QObject(parent), m_window(window), m_vertices(vertices), m_textures(textures), m_rendered(false)
QObject(parent), m_window(window), m_vertices(vertices), m_indices(indices), m_texture(texture), m_drawCount(drawCount), m_rendered(false)
{
connect(m_window, &QQuickWindow::frameSwapped, this, &VncRenderer::frameSwapped);
m_program.addShaderFromSourceFile(QOpenGLShader::Vertex, "draw_shader.vsh");
m_program.addShaderFromSourceFile(QOpenGLShader::Fragment, "draw_shader.fsh");
m_program.bind();
m_program.setUniformValue("texture", 0);
QMatrix4x4 ortho;
ortho.ortho(QRect(QPoint(0, 0), QSize(2048, 2048)));
m_program.setUniformValue("texortho", ortho);
m_program.release();
m_functions = QOpenGLContext::currentContext()->functions();
......@@ -21,36 +26,50 @@ VncRenderer::VncRenderer(QQuickWindow *window,
void VncRenderer::render()
{
/*static QTime frameTime;
qDebug() << qRound(1000.0 / frameTime.elapsed()) << m_textures->length();
qDebug() << qRound(1000.0 / frameTime.elapsed()) << *m_drawCount;
frameTime.restart();*/
m_program.bind();
m_vertices.bind();
if(*m_drawCount > 0) {
m_program.bind();
// m_functions->glActiveTexture(GL_TEXTURE0);
(*m_vertices)->bind();
m_indices.bind();
m_functions->glVertexAttribPointer(m_program.attributeLocation("position"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Vertex),
(void*) 0);
m_functions->glVertexAttribPointer(m_program.attributeLocation("color"),
4,
GL_UNSIGNED_BYTE,
GL_TRUE,
sizeof(Vertex),
(void*) (2 * sizeof(GLushort)));
m_functions->glVertexAttribPointer(m_program.attributeLocation("texcoord"),
2,
GL_UNSIGNED_SHORT,
GL_FALSE,
sizeof(Vertex),
(void*) (2 * sizeof(GLushort) + 4 * sizeof(GLubyte)));
m_program.enableAttributeArray("position");
m_program.enableAttributeArray("color");
m_program.enableAttributeArray("texcoord");
m_functions->glBindTexture(GL_TEXTURE_2D, **m_texture);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
m_program.setAttributeBuffer("position", GL_FLOAT, 0 * sizeof(float), 2, 7 * sizeof(float));
m_program.setAttributeBuffer("color", GL_FLOAT, 2 * sizeof(float), 3, 7 * sizeof(float));
m_program.setAttributeBuffer("texcoord", GL_FLOAT, 5 * sizeof(float), 2, 7 * sizeof(float));
m_program.enableAttributeArray("position");
m_program.enableAttributeArray("color");
m_program.enableAttributeArray("texcoord");
m_functions->glDrawElements(GL_TRIANGLE_STRIP, *m_drawCount, GL_UNSIGNED_SHORT, (void*) 0);
*m_drawCount = 0;
for(int i = 0; i < m_textures->length(); i++) {
m_functions->glBindTexture(GL_TEXTURE_2D, m_textures->at(i));
if(m_textures->at(i) != 0) {
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
m_functions->glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
m_functions->glDrawArrays(GL_TRIANGLE_STRIP, i * 4, 4);
m_window->resetOpenGLState();
}
m_window->resetOpenGLState();
m_rendered = true;
}
QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size)
{
qDebug() << size;
QMatrix4x4 ortho;
ortho.ortho(QRect(QPoint(0, size.height()), QSize(size.width(), -1 * size.height())));
m_program.bind();
......@@ -62,8 +81,8 @@ QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size
#ifdef GL_RGBA8
format.setInternalTextureFormat(GL_RGBA8);
#else
#ifdef GL_RGBA8_OES
format.setInternalTextureFormat(GL_RGBA8_OES);
#ifdef RGBA8_OES
format.setInternalTextureFormat(RGBA8_OES);
#else
format.setInternalTextureFormat(GL_RGBA);
#endif
......@@ -73,19 +92,3 @@ QOpenGLFramebufferObject *VncRenderer::createFramebufferObject(const QSize &size
emit FBOTextureChanged(framebufferObject->texture(), ortho);
return framebufferObject;
}
void VncRenderer::setColorMax(const QVector3D &colorMax)
{
if(m_program.bind()) {
m_program.setUniformValue("colormax", colorMax);
m_program.release();
}
}
void VncRenderer::frameSwapped()
{
if(m_rendered) {
m_rendered = false;
emit finishedRendering();
}
}
......@@ -9,7 +9,6 @@
#include <QMatrix4x4>
#include <QOpenGLShaderProgram>
#include <QOpenGLBuffer>
#include <QSharedPointer>
#include <QTime>
class VncRenderer : public QObject, public QQuickFramebufferObject::Renderer
......@@ -17,24 +16,24 @@ class VncRenderer : public QObject, public QQuickFramebufferObject::Renderer
Q_OBJECT
public:
VncRenderer(QQuickWindow *window,
QOpenGLBuffer vertices,
QSharedPointer<QVector<GLuint>> textures,
QOpenGLBuffer **vertices,
QOpenGLBuffer indices,
GLuint **textures,
unsigned int *drawCount,
QObject *parent = 0);
protected:
void render() override;
QOpenGLFramebufferObject *createFramebufferObject(const QSize &size) override;
private:
QQuickWindow *m_window;
QOpenGLBuffer m_vertices;
QSharedPointer<QVector<GLuint>> m_textures;
QOpenGLBuffer **m_vertices;
QOpenGLBuffer m_indices;
GLuint **m_texture;
unsigned int *m_drawCount;
QVector3D m_colorMax;
QOpenGLShaderProgram m_program;
QOpenGLFunctions *m_functions;
bool m_rendered;
public slots:
void setColorMax(const QVector3D &colorMax);
private slots:
void frameSwapped();
signals:
void FBOTextureChanged(const unsigned int FBOTexture, const QMatrix4x4 &ortho);
void finishedRendering();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment