File openssl-1_1-AES-GCM-performance-optimzation-with-stitched-method.patch of Package openssl-1_1
xxxxxxxxxx1
From 44a563dde1584cd9284e80b6e45ee5019be8d36c Mon Sep 17 00:00:00 20012
From: Danny Tsen <dtsen@us.ibm.com>3
Date: Mon, 18 Oct 2021 10:51:42 -04004
Subject: [PATCH] AES-GCM performance optimzation with stitched method for p9+5
ppc64le6
7
Assembly code reviewed by Shricharan Srivatsan <ssrivat@us.ibm.com>8
9
Reviewed-by: Tomas Mraz <tomas@openssl.org>10
Reviewed-by: Paul Dale <pauli@openssl.org>11
(Merged from https://github.com/openssl/openssl/pull/16854)12
---13
Configurations/00-base-templates.conf | 2 14
crypto/evp/e_aes.c | 33 15
crypto/modes/asm/aes-gcm-ppc.pl | 1439 ++++++++++++++++++++++++++++++++++16
crypto/modes/build.info | 1 17
4 files changed, 1466 insertions(+), 9 deletions(-)18
create mode 100644 crypto/modes/asm/aes-gcm-ppc.pl19
create mode 100644 providers/implementations/ciphers/cipher_aes_gcm_hw_ppc.inc20
21
--- a/Configurations/00-base-templates.conf22
+++ b/Configurations/00-base-templates.conf23
24
bn_asm_src => "bn-ppc.s ppc-mont.s",25
aes_asm_src => "aes_core.c aes_cbc.c aes-ppc.s vpaes-ppc.s aesp8-ppc.s",26
sha1_asm_src => "sha1-ppc.s sha256-ppc.s sha512-ppc.s sha256p8-ppc.s sha512p8-ppc.s",27
- modes_asm_src => "ghashp8-ppc.s",28
+ modes_asm_src => "ghashp8-ppc.s aes-gcm-ppc.s",29
chacha_asm_src => "chacha-ppc.s",30
poly1305_asm_src=> "poly1305-ppc.s poly1305-ppcfp.s",31
},32
--- a/crypto/evp/e_aes.c33
+++ b/crypto/evp/e_aes.c34
35
# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks36
# define HWAES_xts_encrypt aes_p8_xts_encrypt37
# define HWAES_xts_decrypt aes_p8_xts_decrypt38
+# define PPC_AES_GCM_CAPABLE (OPENSSL_ppccap_P & PPC_MADD300)39
+# define AES_GCM_ENC_BYTES 12840
+# define AES_GCM_DEC_BYTES 12841
+size_t ppc_aes_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,42
+ const void *key, unsigned char ivec[16], u64 *Xi);43
+size_t ppc_aes_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,44
+ const void *key, unsigned char ivec[16], u64 *Xi);45
+void gcm_ghash_p8(u64 Xi[2],const u128 Htable[16],const u8 *inp, size_t len);46
+# if PPC_AES_GCM_CAPABLE47
+# define AES_gcm_encrypt ppc_aes_gcm_encrypt48
+# define AES_gcm_decrypt ppc_aes_gcm_decrypt49
+# define AES_GCM_ASM(gctx) ((gctx)->ctr==aes_p8_ctr32_encrypt_blocks && \50
+ (gctx)->gcm.ghash==gcm_ghash_p8)51
+# endif52
#endif53
54
#if defined(OPENSSL_CPUID_OBJ) && ( \55
56
*/57
# define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))58
59
+# define AES_GCM_ENC_BYTES 3260
+# define AES_GCM_DEC_BYTES 1661
+62
int aesni_set_encrypt_key(const unsigned char *userKey, int bits,63
AES_KEY *key);64
int aesni_set_decrypt_key(const unsigned char *userKey, int bits,65
66
if (gctx->ctr) {67
size_t bulk = 0;68
#if defined(AES_GCM_ASM)69
- if (len >= 32 && AES_GCM_ASM(gctx)) {70
+ if (len >= AES_GCM_ENC_BYTES && AES_GCM_ASM(gctx)) {71
if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0))72
return -1;73
74
75
} else {76
size_t bulk = 0;77
#if defined(AES_GCM_ASM2)78
- if (len >= 32 && AES_GCM_ASM2(gctx)) {79
+ if (len >= AES_GCM_ENC_BYTES && AES_GCM_ASM2(gctx)) {80
if (CRYPTO_gcm128_encrypt(&gctx->gcm, NULL, NULL, 0))81
return -1;82
83
84
if (gctx->ctr) {85
size_t bulk = 0;86
#if defined(AES_GCM_ASM)87
- if (len >= 16 && AES_GCM_ASM(gctx)) {88
+ if (len >= AES_GCM_DEC_BYTES && AES_GCM_ASM(gctx)) {89
if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0))90
return -1;91
92
93
} else {94
size_t bulk = 0;95
#if defined(AES_GCM_ASM2)96
- if (len >= 16 && AES_GCM_ASM2(gctx)) {97
+ if (len >= AES_GCM_DEC_BYTES && AES_GCM_ASM2(gctx)) {98
if (CRYPTO_gcm128_decrypt(&gctx->gcm, NULL, NULL, 0))99
return -1;100
101
102
if (gctx->ctr) {103
size_t bulk = 0;104
#if defined(AES_GCM_ASM)105
- if (len >= 32 && AES_GCM_ASM(gctx)) {106
+ if (len >= AES_GCM_ENC_BYTES && AES_GCM_ASM(gctx)) {107
size_t res = (16 - gctx->gcm.mres) % 16;108
109
if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res))110
111
} else {112
size_t bulk = 0;113
#if defined(AES_GCM_ASM2)114
- if (len >= 32 && AES_GCM_ASM2(gctx)) {115
+ if (len >= AES_GCM_ENC_BYTES && AES_GCM_ASM2(gctx)) {116
size_t res = (16 - gctx->gcm.mres) % 16;117
118
if (CRYPTO_gcm128_encrypt(&gctx->gcm, in, out, res))119
120
if (gctx->ctr) {121
size_t bulk = 0;122
#if defined(AES_GCM_ASM)123
- if (len >= 16 && AES_GCM_ASM(gctx)) {124
+ if (len >= AES_GCM_DEC_BYTES && AES_GCM_ASM(gctx)) {125
size_t res = (16 - gctx->gcm.mres) % 16;126
127
if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res))128
129
} else {130
size_t bulk = 0;131
#if defined(AES_GCM_ASM2)132
- if (len >= 16 && AES_GCM_ASM2(gctx)) {133
+ if (len >= AES_GCM_DEC_BYTES && AES_GCM_ASM2(gctx)) {134
size_t res = (16 - gctx->gcm.mres) % 16;135
136
if (CRYPTO_gcm128_decrypt(&gctx->gcm, in, out, res))137
--- /dev/null138
+++ b/crypto/modes/asm/aes-gcm-ppc.pl139
140
+#! /usr/bin/env perl141
+# Copyright 2014-2020 The OpenSSL Project Authors. All Rights Reserved.142
+# Copyright 2021- IBM Inc. All rights reserved143
+#144
+# Licensed under the Apache License 2.0 (the "License"). You may not use145
+# this file except in compliance with the License. You can obtain a copy146
+# in the file LICENSE in the source distribution or at147
+# https://www.openssl.org/source/license.html148
+#149
+#===================================================================================150
+# Written by Danny Tsen <dtsen@us.ibm.com> for OpenSSL Project,151
+#152
+# GHASH is based on the Karatsuba multiplication method.153
+#154
+# Xi xor X1155
+#156
+# X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H =157
+# (X1.h * H4.h + xX.l * H4.l + X1 * H4) +158
+# (X2.h * H3.h + X2.l * H3.l + X2 * H3) +159
+# (X3.h * H2.h + X3.l * H2.l + X3 * H2) +160
+# (X4.h * H.h + X4.l * H.l + X4 * H)161
+#162
+# Xi = v0163
+# H Poly = v2164
+# Hash keys = v3 - v14165
+# ( H.l, H, H.h)166
+# ( H^2.l, H^2, H^2.h)167
+# ( H^3.l, H^3, H^3.h)168
+# ( H^4.l, H^4, H^4.h)169
+#170
+# v30 is IV171
+# v31 - counter 1172
+#173
+# AES used,174
+# vs0 - vs14 for round keys175
+# v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted)176
+#177
+# This implementation uses stitched AES-GCM approach to improve overall performance.178
+# AES is implemented with 8x blocks and GHASH is using 2 4x blocks.179
+#180
+# Current large block (16384 bytes) performance per second with 128 bit key --181
+#182
+# Encrypt Decrypt183
+# Power10[le] (3.5GHz) 5.32G 5.26G184
+#185
+# ===================================================================================186
+#187
+# $output is the last argument if it looks like a file (it has an extension)188
+# $flavour is the first argument if it doesn't look like a file189
+$output = $#ARGV >= 0 && $ARGV[$#ARGV] =~ m|\.\w+$| ? pop : undef;190
+$flavour = $#ARGV >= 0 && $ARGV[0] !~ m|\.| ? shift : undef;191
+192
+if ($flavour =~ /64/) {193
+ $SIZE_T=8;194
+ $LRSAVE=2*$SIZE_T;195
+ $STU="stdu";196
+ $POP="ld";197
+ $PUSH="std";198
+ $UCMP="cmpld";199
+ $SHRI="srdi";200
+} elsif ($flavour =~ /32/) {201
+ $SIZE_T=4;202
+ $LRSAVE=$SIZE_T;203
+ $STU="stwu";204
+ $POP="lwz";205
+ $PUSH="stw";206
+ $UCMP="cmplw";207
+ $SHRI="srwi";208
+} else { die "nonsense $flavour"; }209
+210
+$sp="r1";211
+$FRAME=6*$SIZE_T+13*16; # 13*16 is for v20-v31 offload212
+213
+$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;214
+( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or215
+( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or216
+die "can't locate ppc-xlate.pl";217
+218
+open STDOUT,"| $^X $xlate $flavour \"$output\""219
+ or die "can't call $xlate: $!";220
+221
+$code=<<___;222
+.machine "any"223
+.abiversion 2224
+.text225
+226
+# 4x loops227
+# v15 - v18 - input states228
+# vs1 - vs9 - round keys229
+#230
+.macro Loop_aes_middle4x231
+ xxlor 19+32, 1, 1232
+ xxlor 20+32, 2, 2233
+ xxlor 21+32, 3, 3234
+ xxlor 22+32, 4, 4235
+236
+ vcipher 15, 15, 19237
+ vcipher 16, 16, 19238
+ vcipher 17, 17, 19239
+ vcipher 18, 18, 19240
+241
+ vcipher 15, 15, 20242
+ vcipher 16, 16, 20243
+ vcipher 17, 17, 20244
+ vcipher 18, 18, 20245
+246
+ vcipher 15, 15, 21247
+ vcipher 16, 16, 21248
+ vcipher 17, 17, 21249
+ vcipher 18, 18, 21250
+251
+ vcipher 15, 15, 22252
+ vcipher 16, 16, 22253
+ vcipher 17, 17, 22254
+ vcipher 18, 18, 22255
+256
+ xxlor 19+32, 5, 5257
+ xxlor 20+32, 6, 6258
+ xxlor 21+32, 7, 7259
+ xxlor 22+32, 8, 8260
+261
+ vcipher 15, 15, 19262
+ vcipher 16, 16, 19263
+ vcipher 17, 17, 19264
+ vcipher 18, 18, 19265
+266
+ vcipher 15, 15, 20267
+ vcipher 16, 16, 20268
+ vcipher 17, 17, 20269
+ vcipher 18, 18, 20270
+271
+ vcipher 15, 15, 21272
+ vcipher 16, 16, 21273
+ vcipher 17, 17, 21274
+ vcipher 18, 18, 21275
+276
+ vcipher 15, 15, 22277
+ vcipher 16, 16, 22278
+ vcipher 17, 17, 22279
+ vcipher 18, 18, 22280
+281
+ xxlor 23+32, 9, 9282
+ vcipher 15, 15, 23283
+ vcipher 16, 16, 23284
+ vcipher 17, 17, 23285
+ vcipher 18, 18, 23286
+.endm287
+288
+# 8x loops289
+# v15 - v22 - input states290
+# vs1 - vs9 - round keys291
+#292
+.macro Loop_aes_middle8x293
+ xxlor 23+32, 1, 1294
+ xxlor 24+32, 2, 2295
+ xxlor 25+32, 3, 3296
+ xxlor 26+32, 4, 4297
+298
+ vcipher 15, 15, 23299
+ vcipher 16, 16, 23300
+ vcipher 17, 17, 23301
+ vcipher 18, 18, 23302
+ vcipher 19, 19, 23303
+ vcipher 20, 20, 23304
+ vcipher 21, 21, 23305
+ vcipher 22, 22, 23306
+307
+ vcipher 15, 15, 24308
+ vcipher 16, 16, 24309
+ vcipher 17, 17, 24310
+ vcipher 18, 18, 24311
+ vcipher 19, 19, 24312
+ vcipher 20, 20, 24313
+ vcipher 21, 21, 24314
+ vcipher 22, 22, 24315
+316
+ vcipher 15, 15, 25317
+ vcipher 16, 16, 25318
+ vcipher 17, 17, 25319
+ vcipher 18, 18, 25320
+ vcipher 19, 19, 25321
+ vcipher 20, 20, 25322
+ vcipher 21, 21, 25323
+ vcipher 22, 22, 25324
+325
+ vcipher 15, 15, 26326
+ vcipher 16, 16, 26327
+ vcipher 17, 17, 26328
+ vcipher 18, 18, 26329
+ vcipher 19, 19, 26330
+ vcipher 20, 20, 26331
+ vcipher 21, 21, 26332
+ vcipher 22, 22, 26333
+334
+ xxlor 23+32, 5, 5335
+ xxlor 24+32, 6, 6336
+ xxlor 25+32, 7, 7337
+ xxlor 26+32, 8, 8338
+339
+ vcipher 15, 15, 23340
+ vcipher 16, 16, 23341
+ vcipher 17, 17, 23342
+ vcipher 18, 18, 23343
+ vcipher 19, 19, 23344
+ vcipher 20, 20, 23345
+ vcipher 21, 21, 23346
+ vcipher 22, 22, 23347
+348
+ vcipher 15, 15, 24349
+ vcipher 16, 16, 24350
+ vcipher 17, 17, 24351
+ vcipher 18, 18, 24352
+ vcipher 19, 19, 24353
+ vcipher 20, 20, 24354
+ vcipher 21, 21, 24355
+ vcipher 22, 22, 24356
+357
+ vcipher 15, 15, 25358
+ vcipher 16, 16, 25359
+ vcipher 17, 17, 25360
+ vcipher 18, 18, 25361
+ vcipher 19, 19, 25362
+ vcipher 20, 20, 25363
+ vcipher 21, 21, 25364
+ vcipher 22, 22, 25365
+366
+ vcipher 15, 15, 26367
+ vcipher 16, 16, 26368
+ vcipher 17, 17, 26369
+ vcipher 18, 18, 26370
+ vcipher 19, 19, 26371
+ vcipher 20, 20, 26372
+ vcipher 21, 21, 26373
+ vcipher 22, 22, 26374
+375
+ xxlor 23+32, 9, 9376
+ vcipher 15, 15, 23377
+ vcipher 16, 16, 23378
+ vcipher 17, 17, 23379
+ vcipher 18, 18, 23380
+ vcipher 19, 19, 23381
+ vcipher 20, 20, 23382
+ vcipher 21, 21, 23383
+ vcipher 22, 22, 23384
+.endm385
+386
+#387
+# Compute 4x hash values based on Karatsuba method.388
+#389
+ppc_aes_gcm_ghash:390
+ vxor 15, 15, 0391
+392
+ xxlxor 29, 29, 29393
+394
+ vpmsumd 23, 12, 15 # H4.L * X.L395
+ vpmsumd 24, 9, 16396
+ vpmsumd 25, 6, 17397
+ vpmsumd 26, 3, 18398
+399
+ vxor 23, 23, 24400
+ vxor 23, 23, 25401
+ vxor 23, 23, 26 # L402
+403
+ vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L404
+ vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L405
+ vpmsumd 26, 7, 17406
+ vpmsumd 27, 4, 18407
+408
+ vxor 24, 24, 25409
+ vxor 24, 24, 26410
+ vxor 24, 24, 27 # M411
+412
+ # sum hash and reduction with H Poly413
+ vpmsumd 28, 23, 2 # reduction414
+415
+ xxlor 29+32, 29, 29416
+ vsldoi 26, 24, 29, 8 # mL417
+ vsldoi 29, 29, 24, 8 # mH418
+ vxor 23, 23, 26 # mL + L419
+420
+ vsldoi 23, 23, 23, 8 # swap421
+ vxor 23, 23, 28422
+423
+ vpmsumd 24, 14, 15 # H4.H * X.H424
+ vpmsumd 25, 11, 16425
+ vpmsumd 26, 8, 17426
+ vpmsumd 27, 5, 18427
+428
+ vxor 24, 24, 25429
+ vxor 24, 24, 26430
+ vxor 24, 24, 27431
+432
+ vxor 24, 24, 29433
+434
+ # sum hash and reduction with H Poly435
+ vsldoi 27, 23, 23, 8 # swap436
+ vpmsumd 23, 23, 2437
+ vxor 27, 27, 24438
+ vxor 23, 23, 27439
+440
+ xxlor 32, 23+32, 23+32 # update hash441
+442
+ blr443
+444
+#445
+# Combine two 4x ghash446
+# v15 - v22 - input blocks447
+#448
+.macro ppc_aes_gcm_ghash2_4x449
+ # first 4x hash450
+ vxor 15, 15, 0 # Xi + X451
+452
+ xxlxor 29, 29, 29453
+454
+ vpmsumd 23, 12, 15 # H4.L * X.L455
+ vpmsumd 24, 9, 16456
+ vpmsumd 25, 6, 17457
+ vpmsumd 26, 3, 18458
+459
+ vxor 23, 23, 24460
+ vxor 23, 23, 25461
+ vxor 23, 23, 26 # L462
+463
+ vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L464
+ vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L465
+ vpmsumd 26, 7, 17466
+ vpmsumd 27, 4, 18467
+468
+ vxor 24, 24, 25469
+ vxor 24, 24, 26470
+471
+ # sum hash and reduction with H Poly472
+ vpmsumd 28, 23, 2 # reduction473
+474
+ xxlor 29+32, 29, 29475
+476
+ vxor 24, 24, 27 # M477
+ vsldoi 26, 24, 29, 8 # mL478
+ vsldoi 29, 29, 24, 8 # mH479
+ vxor 23, 23, 26 # mL + L480
+481
+ vsldoi 23, 23, 23, 8 # swap482
+ vxor 23, 23, 28483
+484
+ vpmsumd 24, 14, 15 # H4.H * X.H485
+ vpmsumd 25, 11, 16486
+ vpmsumd 26, 8, 17487
+ vpmsumd 27, 5, 18488
+489
+ vxor 24, 24, 25490
+ vxor 24, 24, 26491
+ vxor 24, 24, 27 # H492
+493
+ vxor 24, 24, 29 # H + mH494
+495
+ # sum hash and reduction with H Poly496
+ vsldoi 27, 23, 23, 8 # swap497
+ vpmsumd 23, 23, 2498
+ vxor 27, 27, 24499
+ vxor 27, 23, 27 # 1st Xi500
+501
+ # 2nd 4x hash502
+ vpmsumd 24, 9, 20503
+ vpmsumd 25, 6, 21504
+ vpmsumd 26, 3, 22505
+ vxor 19, 19, 27 # Xi + X506
+ vpmsumd 23, 12, 19 # H4.L * X.L507
+508
+ vxor 23, 23, 24509
+ vxor 23, 23, 25510
+ vxor 23, 23, 26 # L511
+512
+ vpmsumd 24, 13, 19 # H4.L * X.H + H4.H * X.L513
+ vpmsumd 25, 10, 20 # H3.L * X1.H + H3.H * X1.L514
+ vpmsumd 26, 7, 21515
+ vpmsumd 27, 4, 22516
+517
+ vxor 24, 24, 25518
+ vxor 24, 24, 26519
+520
+ # sum hash and reduction with H Poly521
+ vpmsumd 28, 23, 2 # reduction522
+523
+ xxlor 29+32, 29, 29524
+525
+ vxor 24, 24, 27 # M526
+ vsldoi 26, 24, 29, 8 # mL527
+ vsldoi 29, 29, 24, 8 # mH528
+ vxor 23, 23, 26 # mL + L529
+530
+ vsldoi 23, 23, 23, 8 # swap531
+ vxor 23, 23, 28532
+533
+ vpmsumd 24, 14, 19 # H4.H * X.H534
+ vpmsumd 25, 11, 20535
+ vpmsumd 26, 8, 21536
+ vpmsumd 27, 5, 22537
+538
+ vxor 24, 24, 25539
+ vxor 24, 24, 26540
+ vxor 24, 24, 27 # H541
+542
+ vxor 24, 24, 29 # H + mH543
+544
+ # sum hash and reduction with H Poly545
+ vsldoi 27, 23, 23, 8 # swap546
+ vpmsumd 23, 23, 2547
+ vxor 27, 27, 24548
+ vxor 23, 23, 27549
+550
+ xxlor 32, 23+32, 23+32 # update hash551
+552
+.endm553
+554
+#555
+# Compute update single hash556
+#557
+.macro ppc_update_hash_1x558
+ vxor 28, 28, 0559
+560
+ vxor 19, 19, 19561
+562
+ vpmsumd 22, 3, 28 # L563
+ vpmsumd 23, 4, 28 # M564
+ vpmsumd 24, 5, 28 # H565
+566
+ vpmsumd 27, 22, 2 # reduction567
+568
+ vsldoi 25, 23, 19, 8 # mL569
+ vsldoi 26, 19, 23, 8 # mH570
+ vxor 22, 22, 25 # LL + LL571
+ vxor 24, 24, 26 # HH + HH572
+573
+ vsldoi 22, 22, 22, 8 # swap574
+ vxor 22, 22, 27575
+576
+ vsldoi 20, 22, 22, 8 # swap577
+ vpmsumd 22, 22, 2 # reduction578
+ vxor 20, 20, 24579
+ vxor 22, 22, 20580
+581
+ vmr 0, 22 # update hash582
+583
+.endm584
+585
+#586
+# ppc_aes_gcm_encrypt (const void *inp, void *out, size_t len,587
+# const AES_KEY *key, unsigned char iv[16],588
+# void *Xip);589
+#590
+# r3 - inp591
+# r4 - out592
+# r5 - len593
+# r6 - AES round keys594
+# r7 - iv595
+# r8 - Xi, HPoli, hash keys596
+#597
+.global ppc_aes_gcm_encrypt598
+.align 5599
+ppc_aes_gcm_encrypt:600
+_ppc_aes_gcm_encrypt:601
+602
+ stdu 1,-512(1)603
+ mflr 0604
+605
+ std 14,112(1)606
+ std 15,120(1)607
+ std 16,128(1)608
+ std 17,136(1)609
+ std 18,144(1)610
+ std 19,152(1)611
+ std 20,160(1)612
+ std 21,168(1)613
+ li 9, 256614
+ stvx 20, 9, 1615
+ addi 9, 9, 16616
+ stvx 21, 9, 1617
+ addi 9, 9, 16618
+ stvx 22, 9, 1619
+ addi 9, 9, 16620
+ stvx 23, 9, 1621
+ addi 9, 9, 16622
+ stvx 24, 9, 1623
+ addi 9, 9, 16624
+ stvx 25, 9, 1625
+ addi 9, 9, 16626
+ stvx 26, 9, 1627
+ addi 9, 9, 16628
+ stvx 27, 9, 1629
+ addi 9, 9, 16630
+ stvx 28, 9, 1631
+ addi 9, 9, 16632
+ stvx 29, 9, 1633
+ addi 9, 9, 16634
+ stvx 30, 9, 1635
+ addi 9, 9, 16636
+ stvx 31, 9, 1637
+ std 0, 528(1)638
+639
+ # Load Xi640
+ lxvb16x 32, 0, 8 # load Xi641
+642
+ # load Hash - h^4, h^3, h^2, h643
+ li 10, 32644
+ lxvd2x 2+32, 10, 8 # H Poli645
+ li 10, 48646
+ lxvd2x 3+32, 10, 8 # Hl647
+ li 10, 64648
+ lxvd2x 4+32, 10, 8 # H649
+ li 10, 80650
+ lxvd2x 5+32, 10, 8 # Hh651
+652
+ li 10, 96653
+ lxvd2x 6+32, 10, 8 # H^2l654
+ li 10, 112655
+ lxvd2x 7+32, 10, 8 # H^2656
+ li 10, 128657
+ lxvd2x 8+32, 10, 8 # H^2h658
+659
+ li 10, 144660
+ lxvd2x 9+32, 10, 8 # H^3l661
+ li 10, 160662
+ lxvd2x 10+32, 10, 8 # H^3663
+ li 10, 176664
+ lxvd2x 11+32, 10, 8 # H^3h665
+666
+ li 10, 192667
+ lxvd2x 12+32, 10, 8 # H^4l668
+ li 10, 208669
+ lxvd2x 13+32, 10, 8 # H^4670
+ li 10, 224671
+ lxvd2x 14+32, 10, 8 # H^4h672
+673
+ # initialize ICB: GHASH( IV ), IV - r7674
+ lxvb16x 30+32, 0, 7 # load IV - v30675
+676
+ mr 12, 5 # length677
+ li 11, 0 # block index678
+679
+ # counter 1680
+ vxor 31, 31, 31681
+ vspltisb 22, 1682
+ vsldoi 31, 31, 22,1 # counter 1683
+684
+ # load round key to VSR685
+ lxv 0, 0(6)686
+ lxv 1, 0x10(6)687
+ lxv 2, 0x20(6)688
+ lxv 3, 0x30(6)689
+ lxv 4, 0x40(6)690
+ lxv 5, 0x50(6)691
+ lxv 6, 0x60(6)692
+ lxv 7, 0x70(6)693
+ lxv 8, 0x80(6)694
+ lxv 9, 0x90(6)695
+ lxv 10, 0xa0(6)696
+697
+ # load rounds - 10 (128), 12 (192), 14 (256)698
+ lwz 9,240(6)699
+700
+ #701
+ # vxor state, state, w # addroundkey702
+ xxlor 32+29, 0, 0703
+ vxor 15, 30, 29 # IV + round key - add round key 0704
+705
+ cmpdi 9, 10706
+ beq Loop_aes_gcm_8x707
+708
+ # load 2 more round keys (v11, v12)709
+ lxv 11, 0xb0(6)710
+ lxv 12, 0xc0(6)711
+712
+ cmpdi 9, 12713
+ beq Loop_aes_gcm_8x714
+715
+ # load 2 more round keys (v11, v12, v13, v14)716
+ lxv 13, 0xd0(6)717
+ lxv 14, 0xe0(6)718
+ cmpdi 9, 14719
+ beq Loop_aes_gcm_8x720
+721
+ b aes_gcm_out722
+723
+.align 5724
+Loop_aes_gcm_8x:725
+ mr 14, 3726
+ mr 9, 4727
+728
+ # n blcoks729
+ li 10, 128730
+ divdu 10, 5, 10 # n 128 bytes-blocks731
+ cmpdi 10, 0732
+ beq Loop_last_block733
+734
+ vaddudm 30, 30, 31 # IV + counter735
+ vxor 16, 30, 29736
+ vaddudm 30, 30, 31737
+ vxor 17, 30, 29738
+ vaddudm 30, 30, 31739
+ vxor 18, 30, 29740
+ vaddudm 30, 30, 31741
+ vxor 19, 30, 29742
+ vaddudm 30, 30, 31743
+ vxor 20, 30, 29744
+ vaddudm 30, 30, 31745
+ vxor 21, 30, 29746
+ vaddudm 30, 30, 31747
+ vxor 22, 30, 29748
+749
+ mtctr 10750
+751
+ li 15, 16752
+ li 16, 32753
+ li 17, 48754
+ li 18, 64755
+ li 19, 80756
+ li 20, 96757
+ li 21, 112758
+759
+ lwz 10, 240(6)760
+761
+Loop_8x_block:762
+763
+ lxvb16x 15, 0, 14 # load block764
+ lxvb16x 16, 15, 14 # load block765
+ lxvb16x 17, 16, 14 # load block766
+ lxvb16x 18, 17, 14 # load block767
+ lxvb16x 19, 18, 14 # load block768
+ lxvb16x 20, 19, 14 # load block769
+ lxvb16x 21, 20, 14 # load block770
+ lxvb16x 22, 21, 14 # load block771
+ addi 14, 14, 128772
+773
+ Loop_aes_middle8x774
+775
+ xxlor 23+32, 10, 10776
+777
+ cmpdi 10, 10778
+ beq Do_next_ghash779
+780
+ # 192 bits781
+ xxlor 24+32, 11, 11782
+783
+ vcipher 15, 15, 23784
+ vcipher 16, 16, 23785
+ vcipher 17, 17, 23786
+ vcipher 18, 18, 23787
+ vcipher 19, 19, 23788
+ vcipher 20, 20, 23789
+ vcipher 21, 21, 23790
+ vcipher 22, 22, 23791
+792
+ vcipher 15, 15, 24793
+ vcipher 16, 16, 24794
+ vcipher 17, 17, 24795
+ vcipher 18, 18, 24796
+ vcipher 19, 19, 24797
+ vcipher 20, 20, 24798
+ vcipher 21, 21, 24799
+ vcipher 22, 22, 24800
+801
+ xxlor 23+32, 12, 12802
+803
+ cmpdi 10, 12804
+ beq Do_next_ghash805
+806
+ # 256 bits807
+ xxlor 24+32, 13, 13808
+809
+ vcipher 15, 15, 23810
+ vcipher 16, 16, 23811
+ vcipher 17, 17, 23812
+ vcipher 18, 18, 23813
+ vcipher 19, 19, 23814
+ vcipher 20, 20, 23815
+ vcipher 21, 21, 23816
+ vcipher 22, 22, 23817
+818
+ vcipher 15, 15, 24819
+ vcipher 16, 16, 24820
+ vcipher 17, 17, 24821
+ vcipher 18, 18, 24822
+ vcipher 19, 19, 24823
+ vcipher 20, 20, 24824
+ vcipher 21, 21, 24825
+ vcipher 22, 22, 24826
+827
+ xxlor 23+32, 14, 14828
+829
+ cmpdi 10, 14830
+ beq Do_next_ghash831
+ b aes_gcm_out832
+833
+Do_next_ghash:834
+835
+ #836
+ # last round837
+ vcipherlast 15, 15, 23838
+ vcipherlast 16, 16, 23839
+840
+ xxlxor 47, 47, 15841
+ stxvb16x 47, 0, 9 # store output842
+ xxlxor 48, 48, 16843
+ stxvb16x 48, 15, 9 # store output844
+845
+ vcipherlast 17, 17, 23846
+ vcipherlast 18, 18, 23847
+848
+ xxlxor 49, 49, 17849
+ stxvb16x 49, 16, 9 # store output850
+ xxlxor 50, 50, 18851
+ stxvb16x 50, 17, 9 # store output852
+853
+ vcipherlast 19, 19, 23854
+ vcipherlast 20, 20, 23855
+856
+ xxlxor 51, 51, 19857
+ stxvb16x 51, 18, 9 # store output858
+ xxlxor 52, 52, 20859
+ stxvb16x 52, 19, 9 # store output860
+861
+ vcipherlast 21, 21, 23862
+ vcipherlast 22, 22, 23863
+864
+ xxlxor 53, 53, 21865
+ stxvb16x 53, 20, 9 # store output866
+ xxlxor 54, 54, 22867
+ stxvb16x 54, 21, 9 # store output868
+869
+ addi 9, 9, 128870
+871
+ # ghash here872
+ ppc_aes_gcm_ghash2_4x873
+874
+ xxlor 27+32, 0, 0875
+ vaddudm 30, 30, 31 # IV + counter876
+ vmr 29, 30877
+ vxor 15, 30, 27 # add round key878
+ vaddudm 30, 30, 31879
+ vxor 16, 30, 27880
+ vaddudm 30, 30, 31881
+ vxor 17, 30, 27882
+ vaddudm 30, 30, 31883
+ vxor 18, 30, 27884
+ vaddudm 30, 30, 31885
+ vxor 19, 30, 27886
+ vaddudm 30, 30, 31887
+ vxor 20, 30, 27888
+ vaddudm 30, 30, 31889
+ vxor 21, 30, 27890
+ vaddudm 30, 30, 31891
+ vxor 22, 30, 27892
+893
+ addi 12, 12, -128894
+ addi 11, 11, 128895
+896
+ bdnz Loop_8x_block897
+898
+ vmr 30, 29899
+900
+Loop_last_block:901
+ cmpdi 12, 0902
+ beq aes_gcm_out903
+904
+ # loop last few blocks905
+ li 10, 16906
+ divdu 10, 12, 10907
+908
+ mtctr 10909
+910
+ lwz 10, 240(6)911
+912
+ cmpdi 12, 16913
+ blt Final_block914
+915
+.macro Loop_aes_middle_1x916
+ xxlor 19+32, 1, 1917
+ xxlor 20+32, 2, 2918
+ xxlor 21+32, 3, 3919
+ xxlor 22+32, 4, 4920
+921
+ vcipher 15, 15, 19922
+ vcipher 15, 15, 20923
+ vcipher 15, 15, 21924
+ vcipher 15, 15, 22925
+926
+ xxlor 19+32, 5, 5927
+ xxlor 20+32, 6, 6928
+ xxlor 21+32, 7, 7929
+ xxlor 22+32, 8, 8930
+931
+ vcipher 15, 15, 19932
+ vcipher 15, 15, 20933
+ vcipher 15, 15, 21934
+ vcipher 15, 15, 22935
+936
+ xxlor 19+32, 9, 9937
+ vcipher 15, 15, 19938
+.endm939
+940
+Next_rem_block:941
+ lxvb16x 15, 0, 14 # load block942
+943
+ Loop_aes_middle_1x944
+945
+ xxlor 23+32, 10, 10946
+947
+ cmpdi 10, 10948
+ beq Do_next_1x949
+950
+ # 192 bits951
+ xxlor 24+32, 11, 11952
+953
+ vcipher 15, 15, 23954
+ vcipher 15, 15, 24955
+956
+ xxlor 23+32, 12, 12957
+958
+ cmpdi 10, 12959
+ beq Do_next_1x960
+961
+ # 256 bits962
+ xxlor 24+32, 13, 13963
+964
+ vcipher 15, 15, 23965
+ vcipher 15, 15, 24966
+967
+ xxlor 23+32, 14, 14968
+969
+ cmpdi 10, 14970
+ beq Do_next_1x971
+972
+Do_next_1x:973
+ vcipherlast 15, 15, 23974
+975
+ xxlxor 47, 47, 15976
+ stxvb16x 47, 0, 9 # store output977
+ addi 14, 14, 16978
+ addi 9, 9, 16979
+980
+ vmr 28, 15981
+ ppc_update_hash_1x982
+983
+ addi 12, 12, -16984
+ addi 11, 11, 16985
+ xxlor 19+32, 0, 0986
+ vaddudm 30, 30, 31 # IV + counter987
+ vxor 15, 30, 19 # add round key988
+989
+ bdnz Next_rem_block990
+991
+ cmpdi 12, 0992
+ beq aes_gcm_out993
+994
+Final_block:995
+ Loop_aes_middle_1x996
+997
+ xxlor 23+32, 10, 10998
+999
+ cmpdi 10, 101000
+ beq Do_final_1x1001
+1002
+ # 192 bits1003
+ xxlor 24+32, 11, 111004
+1005
+ vcipher 15, 15, 231006
+ vcipher 15, 15, 241007
+1008
+ xxlor 23+32, 12, 121009
+1010
+ cmpdi 10, 121011
+ beq Do_final_1x1012
+1013
+ # 256 bits1014
+ xxlor 24+32, 13, 131015
+1016
+ vcipher 15, 15, 231017
+ vcipher 15, 15, 241018
+1019
+ xxlor 23+32, 14, 141020
+1021
+ cmpdi 10, 141022
+ beq Do_final_1x1023
+1024
+Do_final_1x:1025
+ vcipherlast 15, 15, 231026
+1027
+ lxvb16x 15, 0, 14 # load last block1028
+ xxlxor 47, 47, 151029
+1030
+ # create partial block mask1031
+ li 15, 161032
+ sub 15, 15, 12 # index to the mask1033
+1034
+ vspltisb 16, -1 # first 16 bytes - 0xffff...ff1035
+ vspltisb 17, 0 # second 16 bytes - 0x0000...001036
+ li 10, 1921037
+ stvx 16, 10, 11038
+ addi 10, 10, 161039
+ stvx 17, 10, 11040
+1041
+ addi 10, 1, 1921042
+ lxvb16x 16, 15, 10 # load partial block mask1043
+ xxland 47, 47, 161044
+1045
+ vmr 28, 151046
+ ppc_update_hash_1x1047
+1048
+ # * should store only the remaining bytes.1049
+ bl Write_partial_block1050
+1051
+ b aes_gcm_out1052
+1053
+#1054
+# Write partial block1055
+# r9 - output1056
+# r12 - remaining bytes1057
+# v15 - partial input data1058
+#1059
+Write_partial_block:1060
+ li 10, 1921061
+ stxvb16x 15+32, 10, 1 # last block1062
+1063
+ #add 10, 9, 11 # Output1064
+ addi 10, 9, -11065
+ addi 16, 1, 1911066
+1067
+ mtctr 12 # remaining bytes1068
+ li 15, 01069
+1070
+Write_last_byte:1071
+ lbzu 14, 1(16)1072
+ stbu 14, 1(10)1073
+ bdnz Write_last_byte1074
+ blr1075
+1076
+aes_gcm_out:1077
+ # out = state1078
+ stxvb16x 32, 0, 8 # write out Xi1079
+ add 3, 11, 12 # return count1080
+1081
+ li 9, 2561082
+ lvx 20, 9, 11083
+ addi 9, 9, 161084
+ lvx 21, 9, 11085
+ addi 9, 9, 161086
+ lvx 22, 9, 11087
+ addi 9, 9, 161088
+ lvx 23, 9, 11089
+ addi 9, 9, 161090
+ lvx 24, 9, 11091
+ addi 9, 9, 161092
+ lvx 25, 9, 11093
+ addi 9, 9, 161094
+ lvx 26, 9, 11095
+ addi 9, 9, 161096
+ lvx 27, 9, 11097
+ addi 9, 9, 161098
+ lvx 28, 9, 11099
+ addi 9, 9, 161100
+ lvx 29, 9, 11101
+ addi 9, 9, 161102
+ lvx 30, 9, 11103
+ addi 9, 9, 161104
+ lvx 31, 9, 11105
+1106
+ ld 0, 528(1)1107
+ ld 14,112(1)1108
+ ld 15,120(1)1109
+ ld 16,128(1)1110
+ ld 17,136(1)1111
+ ld 18,144(1)1112
+ ld 19,152(1)1113
+ ld 20,160(1)1114
+ ld 21,168(1)1115
+1116
+ mtlr 01117
+ addi 1, 1, 5121118
+ blr1119
+1120
+#1121
+# 8x Decrypt1122
+#1123
+.global ppc_aes_gcm_decrypt1124
+.align 51125
+ppc_aes_gcm_decrypt:1126
+_ppc_aes_gcm_decrypt:1127
+1128
+ stdu 1,-512(1)1129
+ mflr 01130
+1131
+ std 14,112(1)1132
+ std 15,120(1)1133
+ std 16,128(1)1134
+ std 17,136(1)1135
+ std 18,144(1)1136
+ std 19,152(1)1137
+ std 20,160(1)1138
+ std 21,168(1)1139
+ li 9, 2561140
+ stvx 20, 9, 11141
+ addi 9, 9, 161142
+ stvx 21, 9, 11143
+ addi 9, 9, 161144
+ stvx 22, 9, 11145
+ addi 9, 9, 161146
+ stvx 23, 9, 11147
+ addi 9, 9, 161148
+ stvx 24, 9, 11149
+ addi 9, 9, 161150
+ stvx 25, 9, 11151
+ addi 9, 9, 161152
+ stvx 26, 9, 11153
+ addi 9, 9, 161154
+ stvx 27, 9, 11155
+ addi 9, 9, 161156
+ stvx 28, 9, 11157
+ addi 9, 9, 161158
+ stvx 29, 9, 11159
+ addi 9, 9, 161160
+ stvx 30, 9, 11161
+ addi 9, 9, 161162
+ stvx 31, 9, 11163
+ std 0, 528(1)1164
+1165
+ # Load Xi1166
+ lxvb16x 32, 0, 8 # load Xi1167
+1168
+ # load Hash - h^4, h^3, h^2, h1169
+ li 10, 321170
+ lxvd2x 2+32, 10, 8 # H Poli1171
+ li 10, 481172
+ lxvd2x 3+32, 10, 8 # Hl1173
+ li 10, 641174
+ lxvd2x 4+32, 10, 8 # H1175
+ li 10, 801176
+ lxvd2x 5+32, 10, 8 # Hh1177
+1178
+ li 10, 961179
+ lxvd2x 6+32, 10, 8 # H^2l1180
+ li 10, 1121181
+ lxvd2x 7+32, 10, 8 # H^21182
+ li 10, 1281183
+ lxvd2x 8+32, 10, 8 # H^2h1184
+1185
+ li 10, 1441186
+ lxvd2x 9+32, 10, 8 # H^3l1187
+ li 10, 1601188
+ lxvd2x 10+32, 10, 8 # H^31189
+ li 10, 1761190
+ lxvd2x 11+32, 10, 8 # H^3h1191
+1192
+ li 10, 1921193
+ lxvd2x 12+32, 10, 8 # H^4l1194
+ li 10, 2081195
+ lxvd2x 13+32, 10, 8 # H^41196
+ li 10, 2241197
+ lxvd2x 14+32, 10, 8 # H^4h1198
+1199
+ # initialize ICB: GHASH( IV ), IV - r71200
+ lxvb16x 30+32, 0, 7 # load IV - v301201
+1202
+ mr 12, 5 # length1203
+ li 11, 0 # block index1204
+1205
+ # counter 11206
+ vxor 31, 31, 311207
+ vspltisb 22, 11208
+ vsldoi 31, 31, 22,1 # counter 11209
+1210
+ # load round key to VSR1211
+ lxv 0, 0(6)1212
+ lxv 1, 0x10(6)1213
+ lxv 2, 0x20(6)1214
+ lxv 3, 0x30(6)1215
+ lxv 4, 0x40(6)1216
+ lxv 5, 0x50(6)1217
+ lxv 6, 0x60(6)1218
+ lxv 7, 0x70(6)1219
+ lxv 8, 0x80(6)1220
+ lxv 9, 0x90(6)1221
+ lxv 10, 0xa0(6)1222
+1223
+ # load rounds - 10 (128), 12 (192), 14 (256)1224
+ lwz 9,240(6)1225
+1226
+ #1227
+ # vxor state, state, w # addroundkey1228
+ xxlor 32+29, 0, 01229
+ vxor 15, 30, 29 # IV + round key - add round key 01230
+1231
+ cmpdi 9, 101232
+ beq Loop_aes_gcm_8x_dec1233
+1234
+ # load 2 more round keys (v11, v12)1235
+ lxv 11, 0xb0(6)1236
+ lxv 12, 0xc0(6)1237
+1238
+ cmpdi 9, 121239
+ beq Loop_aes_gcm_8x_dec1240
+1241
+ # load 2 more round keys (v11, v12, v13, v14)1242
+ lxv 13, 0xd0(6)1243
+ lxv 14, 0xe0(6)1244
+ cmpdi 9, 141245
+ beq Loop_aes_gcm_8x_dec1246
+1247
+ b aes_gcm_out1248
+1249
+.align 51250
+Loop_aes_gcm_8x_dec:1251
+ mr 14, 31252
+ mr 9, 41253
+1254
+ # n blcoks1255
+ li 10, 1281256
+ divdu 10, 5, 10 # n 128 bytes-blocks1257
+ cmpdi 10, 01258
+ beq Loop_last_block_dec1259
+1260
+ vaddudm 30, 30, 31 # IV + counter1261
+ vxor 16, 30, 291262
+ vaddudm 30, 30, 311263
+ vxor 17, 30, 291264
+ vaddudm 30, 30, 311265
+ vxor 18, 30, 291266
+ vaddudm 30, 30, 311267
+ vxor 19, 30, 291268
+ vaddudm 30, 30, 311269
+ vxor 20, 30, 291270
+ vaddudm 30, 30, 311271
+ vxor 21, 30, 291272
+ vaddudm 30, 30, 311273
+ vxor 22, 30, 291274
+1275
+ mtctr 101276
+1277
+ li 15, 161278
+ li 16, 321279
+ li 17, 481280
+ li 18, 641281
+ li 19, 801282
+ li 20, 961283
+ li 21, 1121284
+1285
+ lwz 10, 240(6)1286
+1287
+Loop_8x_block_dec:1288
+1289
+ lxvb16x 15, 0, 14 # load block1290
+ lxvb16x 16, 15, 14 # load block1291
+ lxvb16x 17, 16, 14 # load block1292
+ lxvb16x 18, 17, 14 # load block1293
+ lxvb16x 19, 18, 14 # load block1294
+ lxvb16x 20, 19, 14 # load block1295
+ lxvb16x 21, 20, 14 # load block1296
+ lxvb16x 22, 21, 14 # load block1297
+ addi 14, 14, 1281298
+1299
+ Loop_aes_middle8x1300
+1301
+ xxlor 23+32, 10, 101302
+1303
+ cmpdi 10, 101304
+ beq Do_last_aes_dec1305
+1306
+ # 192 bits1307
+ xxlor 24+32, 11, 111308
+1309
+ vcipher 15, 15, 231310
+ vcipher 16, 16, 231311
+ vcipher 17, 17, 231312
+ vcipher 18, 18, 231313
+ vcipher 19, 19, 231314
+ vcipher 20, 20, 231315
+ vcipher 21, 21, 231316
+ vcipher 22, 22, 231317
+1318
+ vcipher 15, 15, 241319
+ vcipher 16, 16, 241320
+ vcipher 17, 17, 241321
+ vcipher 18, 18, 241322
+ vcipher 19, 19, 241323
+ vcipher 20, 20, 241324
+ vcipher 21, 21, 241325
+ vcipher 22, 22, 241326
+1327
+ xxlor 23+32, 12, 121328
+1329
+ cmpdi 10, 121330
+ beq Do_last_aes_dec1331
+1332
+ # 256 bits1333
+ xxlor 24+32, 13, 131334
+1335
+ vcipher 15, 15, 231336
+ vcipher 16, 16, 231337
+ vcipher 17, 17, 231338
+ vcipher 18, 18, 231339
+ vcipher 19, 19, 231340
+ vcipher 20, 20, 231341
+ vcipher 21, 21, 231342
+ vcipher 22, 22, 231343
+1344
+ vcipher 15, 15, 241345
+ vcipher 16, 16, 241346
+ vcipher 17, 17, 241347
+ vcipher 18, 18, 241348
+ vcipher 19, 19, 241349
+ vcipher 20, 20, 241350
+ vcipher 21, 21, 241351
+ vcipher 22, 22, 241352
+1353
+ xxlor 23+32, 14, 141354
+1355
+ cmpdi 10, 141356
+ beq Do_last_aes_dec1357
+ b aes_gcm_out1358
+1359
+Do_last_aes_dec:1360
+1361
+ #1362
+ # last round1363
+ vcipherlast 15, 15, 231364
+ vcipherlast 16, 16, 231365
+1366
+ xxlxor 47, 47, 151367
+ stxvb16x 47, 0, 9 # store output1368
+ xxlxor 48, 48, 161369
+ stxvb16x 48, 15, 9 # store output1370
+1371
+ vcipherlast 17, 17, 231372
+ vcipherlast 18, 18, 231373
+1374
+ xxlxor 49, 49, 171375
+ stxvb16x 49, 16, 9 # store output1376
+ xxlxor 50, 50, 181377
+ stxvb16x 50, 17, 9 # store output1378
+1379
+ vcipherlast 19, 19, 231380
+ vcipherlast 20, 20, 231381
+1382
+ xxlxor 51, 51, 191383
+ stxvb16x 51, 18, 9 # store output1384
+ xxlxor 52, 52, 201385
+ stxvb16x 52, 19, 9 # store output1386
+1387
+ vcipherlast 21, 21, 231388
+ vcipherlast 22, 22, 231389
+1390
+ xxlxor 53, 53, 211391
+ stxvb16x 53, 20, 9 # store output1392
+ xxlxor 54, 54, 221393
+ stxvb16x 54, 21, 9 # store output1394
+1395
+ addi 9, 9, 1281396
+1397
+ xxlor 15+32, 15, 151398
+ xxlor 16+32, 16, 161399
+ xxlor 17+32, 17, 171400
+ xxlor 18+32, 18, 181401
+ xxlor 19+32, 19, 191402
+ xxlor 20+32, 20, 201403
+ xxlor 21+32, 21, 211404
+ xxlor 22+32, 22, 221405
+1406
+ # ghash here1407
+ ppc_aes_gcm_ghash2_4x1408
+1409
+ xxlor 27+32, 0, 01410
+ vaddudm 30, 30, 31 # IV + counter1411
+ vmr 29, 301412
+ vxor 15, 30, 27 # add round key1413
+ vaddudm 30, 30, 311414
+ vxor 16, 30, 271415
+ vaddudm 30, 30, 311416
+ vxor 17, 30, 271417
+ vaddudm 30, 30, 311418
+ vxor 18, 30, 271419
+ vaddudm 30, 30, 311420
+ vxor 19, 30, 271421
+ vaddudm 30, 30, 311422
+ vxor 20, 30, 271423
+ vaddudm 30, 30, 311424
+ vxor 21, 30, 271425
+ vaddudm 30, 30, 311426
+ vxor 22, 30, 271427
+ addi 12, 12, -1281428
+ addi 11, 11, 1281429
+1430
+ bdnz Loop_8x_block_dec1431
+1432
+ vmr 30, 291433
+1434
+Loop_last_block_dec:1435
+ cmpdi 12, 01436
+ beq aes_gcm_out1437
+1438
+ # loop last few blocks1439
+ li 10, 161440
+ divdu 10, 12, 101441
+1442
+ mtctr 101443
+1444
+ lwz 10,240(6)1445
+1446
+ cmpdi 12, 161447
+ blt Final_block_dec1448
+1449
+Next_rem_block_dec:1450
+ lxvb16x 15, 0, 14 # load block1451
+1452
+ Loop_aes_middle_1x1453
+1454
+ xxlor 23+32, 10, 101455
+1456
+ cmpdi 10, 101457
+ beq Do_next_1x_dec1458
+1459
+ # 192 bits1460
+ xxlor 24+32, 11, 111461
+1462
+ vcipher 15, 15, 231463
+ vcipher 15, 15, 241464
+1465
+ xxlor 23+32, 12, 121466
+1467
+ cmpdi 10, 121468
+ beq Do_next_1x_dec1469
+1470
+ # 256 bits1471
+ xxlor 24+32, 13, 131472
+1473
+ vcipher 15, 15, 231474
+ vcipher 15, 15, 241475
+1476
+ xxlor 23+32, 14, 141477
+1478
+ cmpdi 10, 141479
+ beq Do_next_1x_dec1480
+1481
+Do_next_1x_dec:1482
+ vcipherlast 15, 15, 231483
+1484
+ xxlxor 47, 47, 151485
+ stxvb16x 47, 0, 9 # store output1486
+ addi 14, 14, 161487
+ addi 9, 9, 161488
+1489
+ xxlor 28+32, 15, 151490
+ ppc_update_hash_1x1491
+1492
+ addi 12, 12, -161493
+ addi 11, 11, 161494
+ xxlor 19+32, 0, 01495
+ vaddudm 30, 30, 31 # IV + counter1496
+ vxor 15, 30, 19 # add round key1497
+1498
+ bdnz Next_rem_block_dec1499
+1500
+ cmpdi 12, 01501
+ beq aes_gcm_out1502
+1503
+Final_block_dec:1504
+ Loop_aes_middle_1x1505
+1506
+ xxlor 23+32, 10, 101507
+1508
+ cmpdi 10, 101509
+ beq Do_final_1x_dec1510
+1511
+ # 192 bits1512
+ xxlor 24+32, 11, 111513
+1514
+ vcipher 15, 15, 231515
+ vcipher 15, 15, 241516
+1517
+ xxlor 23+32, 12, 121518
+1519
+ cmpdi 10, 121520
+ beq Do_final_1x_dec1521
+1522
+ # 256 bits1523
+ xxlor 24+32, 13, 131524
+1525
+ vcipher 15, 15, 231526
+ vcipher 15, 15, 241527
+1528
+ xxlor 23+32, 14, 141529
+1530
+ cmpdi 10, 141531
+ beq Do_final_1x_dec1532
+1533
+Do_final_1x_dec:1534
+ vcipherlast 15, 15, 231535
+1536
+ lxvb16x 15, 0, 14 # load block1537
+ xxlxor 47, 47, 151538
+1539
+ # create partial block mask1540
+ li 15, 161541
+ sub 15, 15, 12 # index to the mask1542
+1543
+ vspltisb 16, -1 # first 16 bytes - 0xffff...ff1544
+ vspltisb 17, 0 # second 16 bytes - 0x0000...001545
+ li 10, 1921546
+ stvx 16, 10, 11547
+ addi 10, 10, 161548
+ stvx 17, 10, 11549
+1550
+ addi 10, 1, 1921551
+ lxvb16x 16, 15, 10 # load block mask1552
+ xxland 47, 47, 161553
+1554
+ xxlor 28+32, 15, 151555
+ ppc_update_hash_1x1556
+1557
+ # * should store only the remaining bytes.1558
+ bl Write_partial_block1559
+1560
+ b aes_gcm_out1561
+1562
+1563
+___1564
+1565
+foreach (split("\n",$code)) {1566
+ s/\`([^\`]*)\`/eval $1/geo;1567
+1568
+ if ($flavour =~ /le$/o) { # little-endian1569
+ s/le\?//o or1570
+ s/be\?/#be#/o;1571
+ } else {1572
+ s/le\?/#le#/o or1573
+ s/be\?//o;1574
+ }1575
+ print $_,"\n";1576
+}1577
+1578
+close STDOUT or die "error closing STDOUT: $!"; # enforce flush1579
--- a/crypto/modes/build.info1580
+++ b/crypto/modes/build.info1581
1582
GENERATE[ghash-alpha.S]=asm/ghash-alpha.pl $(PERLASM_SCHEME)1583
GENERATE[ghash-parisc.s]=asm/ghash-parisc.pl $(PERLASM_SCHEME)1584
GENERATE[ghashp8-ppc.s]=asm/ghashp8-ppc.pl $(PERLASM_SCHEME)1585
+GENERATE[aes-gcm-ppc.s]=asm/aes-gcm-ppc.pl $(PERLASM_SCHEME)1586
GENERATE[ghash-armv4.S]=asm/ghash-armv4.pl $(PERLASM_SCHEME)1587
INCLUDE[ghash-armv4.o]=..1588
GENERATE[ghashv8-armx.S]=asm/ghashv8-armx.pl $(PERLASM_SCHEME)1589