[spice] relocate openssl code into seperate function

This is in preperation of switching to an alternative SSL library
as OpenSSL conflicts with the GNU licence.
This commit is contained in:
Geoffrey McRae 2018-05-22 09:39:03 +10:00
parent 6e0eac0abc
commit cba6630aa0
4 changed files with 112 additions and 32 deletions

View file

@ -17,6 +17,7 @@ CFLAGS += -DBUILD_VERSION='"$(shell git describe --always --long --dirty --abbr
OBJS = main.o \
lg-renderer.o \
spice/rsa.o \
spice/spice.o \
parsers/nal.o \
decoders/null.o \

74
client/spice/rsa.c Normal file
View file

@ -0,0 +1,74 @@
/*
Looking Glass - KVM FrameRelay (KVMFR) Client
Copyright (C) 2017 Geoffrey McRae <geoff@hostfission.com>
https://looking-glass.hostfission.com
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include "rsa.h"
#include "debug.h"
#include <spice/protocol.h>
#include <string.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/x509.h>
bool spice_rsa_encrypt_password(uint8_t * pub_key, char * password, struct spice_password * result)
{
BIO *bioKey = BIO_new(BIO_s_mem());
if (!bioKey)
{
DEBUG_ERROR("failed to allocate bioKey");
return false;
}
BIO_write(bioKey, pub_key, SPICE_TICKET_PUBKEY_BYTES);
EVP_PKEY *rsaKey = d2i_PUBKEY_bio(bioKey, NULL);
RSA *rsa = EVP_PKEY_get1_RSA(rsaKey);
result->size = RSA_size(rsa);
result->data = (char *)malloc(result->size);
if (RSA_public_encrypt(
strlen(password) + 1,
(uint8_t*)password,
(uint8_t*)result->data,
rsa,
RSA_PKCS1_OAEP_PADDING
) <= 0)
{
free(result->data);
result->size = 0;
result->data = NULL;
DEBUG_ERROR("rsa public encrypt failed");
EVP_PKEY_free(rsaKey);
BIO_free(bioKey);
return false;
}
EVP_PKEY_free(rsaKey);
BIO_free(bioKey);
return true;
}
void spice_rsa_free_password(struct spice_password * pass)
{
free(pass->data);
pass->size = 0;
pass->data = NULL;
}

30
client/spice/rsa.h Normal file
View file

@ -0,0 +1,30 @@
/*
Looking Glass - KVM FrameRelay (KVMFR) Client
Copyright (C) 2017 Geoffrey McRae <geoff@hostfission.com>
https://looking-glass.hostfission.com
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 59 Temple
Place, Suite 330, Boston, MA 02111-1307 USA
*/
#include <stdbool.h>
#include <stdint.h>
struct spice_password
{
char * data;
unsigned int size;
};
bool spice_rsa_encrypt_password(uint8_t * pub_key, char * password, struct spice_password * result);
void spice_rsa_free_password(struct spice_password * pass);

View file

@ -27,10 +27,6 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <stdio.h>
#include <stdint.h>
#include <openssl/rsa.h>
#include <openssl/evp.h>
#include <openssl/x509.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/select.h>
@ -42,6 +38,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA
#include <spice/error_codes.h>
#include "messages.h"
#include "rsa.h"
#ifdef DEBUG_SPICE_MOUSE
#define DEBUG_MOUSE(fmt, args...) DEBUG_PRINT("[M]", fmt, ##args)
@ -584,45 +581,23 @@ bool spice_connect_channel(struct SpiceChannel * channel)
spice_read(channel, &capsCommon , sizeof(capsCommon ));
spice_read(channel, &capsChannel, sizeof(capsChannel));
BIO *bioKey = BIO_new(BIO_s_mem());
if (!bioKey)
struct spice_password pass;
if (!spice_rsa_encrypt_password(reply.pub_key, spice.password, &pass))
{
DEBUG_ERROR("failed to allocate bioKey");
spice_disconnect_channel(channel);
return false;
}
BIO_write(bioKey, reply.pub_key, SPICE_TICKET_PUBKEY_BYTES);
EVP_PKEY *rsaKey = d2i_PUBKEY_bio(bioKey, NULL);
RSA *rsa = EVP_PKEY_get1_RSA(rsaKey);
char enc[RSA_size(rsa)];
if (RSA_public_encrypt(
strlen(spice.password) + 1,
(uint8_t*)spice.password,
(uint8_t*)enc,
rsa,
RSA_PKCS1_OAEP_PADDING
) <= 0)
{
DEBUG_ERROR("rsa public encrypt failed");
spice_disconnect_channel(channel);
EVP_PKEY_free(rsaKey);
BIO_free(bioKey);
return false;
}
ssize_t rsaSize = RSA_size(rsa);
EVP_PKEY_free(rsaKey);
BIO_free(bioKey);
if (!spice_write(channel, enc, rsaSize))
if (!spice_write(channel, pass.data, pass.size))
{
spice_rsa_free_password(&pass);
DEBUG_ERROR("failed to write encrypted data");
spice_disconnect_channel(channel);
return false;
}
spice_rsa_free_password(&pass);
uint32_t linkResult;
if (!spice_read(channel, &linkResult, sizeof(linkResult)))
{