001 /**
002 * Licensed to the Apache Software Foundation (ASF) under one
003 * or more contributor license agreements. See the NOTICE file
004 * distributed with this work for additional information
005 * regarding copyright ownership. The ASF licenses this file
006 * to you under the Apache License, Version 2.0 (the
007 * "License"); you may not use this file except in compliance
008 * with the License. You may obtain a copy of the License at
009 *
010 * http://www.apache.org/licenses/LICENSE-2.0
011 *
012 * Unless required by applicable law or agreed to in writing, software
013 * distributed under the License is distributed on an "AS IS" BASIS,
014 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
015 * See the License for the specific language governing permissions and
016 * limitations under the License.
017 */
018
019 package org.apache.hadoop.crypto.key;
020
021 import java.io.ByteArrayInputStream;
022 import java.io.ByteArrayOutputStream;
023 import java.io.IOException;
024 import java.io.InputStreamReader;
025 import java.io.OutputStreamWriter;
026 import java.security.NoSuchAlgorithmException;
027 import java.util.Collections;
028 import java.util.Date;
029 import java.util.HashMap;
030 import java.util.List;
031 import java.util.Map;
032
033 import com.google.gson.stream.JsonReader;
034 import com.google.gson.stream.JsonWriter;
035 import org.apache.hadoop.classification.InterfaceAudience;
036 import org.apache.hadoop.classification.InterfaceStability;
037 import org.apache.hadoop.conf.Configuration;
038
039 import javax.crypto.KeyGenerator;
040
041 /**
042 * A provider of secret key material for Hadoop applications. Provides an
043 * abstraction to separate key storage from users of encryption. It
044 * is intended to support getting or storing keys in a variety of ways,
045 * including third party bindings.
046 * <P/>
047 * <code>KeyProvider</code> implementations must be thread safe.
048 */
049 @InterfaceAudience.Public
050 @InterfaceStability.Unstable
051 public abstract class KeyProvider {
052 public static final String DEFAULT_CIPHER_NAME =
053 "hadoop.security.key.default.cipher";
054 public static final String DEFAULT_CIPHER = "AES/CTR/NoPadding";
055 public static final String DEFAULT_BITLENGTH_NAME =
056 "hadoop.security.key.default.bitlength";
057 public static final int DEFAULT_BITLENGTH = 128;
058
059 private final Configuration conf;
060
061 /**
062 * The combination of both the key version name and the key material.
063 */
064 public static class KeyVersion {
065 private final String name;
066 private final String versionName;
067 private final byte[] material;
068
069 protected KeyVersion(String name, String versionName,
070 byte[] material) {
071 this.name = name;
072 this.versionName = versionName;
073 this.material = material;
074 }
075
076 public String getName() {
077 return name;
078 }
079
080 public String getVersionName() {
081 return versionName;
082 }
083
084 public byte[] getMaterial() {
085 return material;
086 }
087
088 public String toString() {
089 StringBuilder buf = new StringBuilder();
090 buf.append("key(");
091 buf.append(versionName);
092 buf.append(")=");
093 if (material == null) {
094 buf.append("null");
095 } else {
096 for(byte b: material) {
097 buf.append(' ');
098 int right = b & 0xff;
099 if (right < 0x10) {
100 buf.append('0');
101 }
102 buf.append(Integer.toHexString(right));
103 }
104 }
105 return buf.toString();
106 }
107 }
108
109 /**
110 * Key metadata that is associated with the key.
111 */
112 public static class Metadata {
113 private final static String CIPHER_FIELD = "cipher";
114 private final static String BIT_LENGTH_FIELD = "bitLength";
115 private final static String CREATED_FIELD = "created";
116 private final static String DESCRIPTION_FIELD = "description";
117 private final static String VERSIONS_FIELD = "versions";
118 private final static String ATTRIBUTES_FIELD = "attributes";
119
120 private final String cipher;
121 private final int bitLength;
122 private final String description;
123 private final Date created;
124 private int versions;
125 private Map<String, String> attributes;
126
127 protected Metadata(String cipher, int bitLength, String description,
128 Map<String, String> attributes, Date created, int versions) {
129 this.cipher = cipher;
130 this.bitLength = bitLength;
131 this.description = description;
132 this.attributes = (attributes == null || attributes.isEmpty())
133 ? null : attributes;
134 this.created = created;
135 this.versions = versions;
136 }
137
138 public String toString() {
139 final StringBuilder metaSB = new StringBuilder();
140 metaSB.append("cipher: ").append(cipher).append(", ");
141 metaSB.append("length: ").append(bitLength).append(", ");
142 metaSB.append("description: ").append(description).append(", ");
143 metaSB.append("created: ").append(created).append(", ");
144 metaSB.append("version: ").append(versions).append(", ");
145 metaSB.append("attributes: ");
146 if ((attributes != null) && !attributes.isEmpty()) {
147 for (Map.Entry<String, String> attribute : attributes.entrySet()) {
148 metaSB.append("[");
149 metaSB.append(attribute.getKey());
150 metaSB.append("=");
151 metaSB.append(attribute.getValue());
152 metaSB.append("], ");
153 }
154 metaSB.deleteCharAt(metaSB.length() - 2); // remove last ', '
155 } else {
156 metaSB.append("null");
157 }
158 return metaSB.toString();
159 }
160
161 public String getDescription() {
162 return description;
163 }
164
165 public Date getCreated() {
166 return created;
167 }
168
169 public String getCipher() {
170 return cipher;
171 }
172
173 @SuppressWarnings("unchecked")
174 public Map<String, String> getAttributes() {
175 return (attributes == null) ? Collections.EMPTY_MAP : attributes;
176 }
177
178 /**
179 * Get the algorithm from the cipher.
180 * @return the algorithm name
181 */
182 public String getAlgorithm() {
183 int slash = cipher.indexOf('/');
184 if (slash == - 1) {
185 return cipher;
186 } else {
187 return cipher.substring(0, slash);
188 }
189 }
190
191 public int getBitLength() {
192 return bitLength;
193 }
194
195 public int getVersions() {
196 return versions;
197 }
198
199 protected int addVersion() {
200 return versions++;
201 }
202
203 /**
204 * Serialize the metadata to a set of bytes.
205 * @return the serialized bytes
206 * @throws IOException
207 */
208 protected byte[] serialize() throws IOException {
209 ByteArrayOutputStream buffer = new ByteArrayOutputStream();
210 JsonWriter writer = new JsonWriter(new OutputStreamWriter(buffer));
211 try {
212 writer.beginObject();
213 if (cipher != null) {
214 writer.name(CIPHER_FIELD).value(cipher);
215 }
216 if (bitLength != 0) {
217 writer.name(BIT_LENGTH_FIELD).value(bitLength);
218 }
219 if (created != null) {
220 writer.name(CREATED_FIELD).value(created.getTime());
221 }
222 if (description != null) {
223 writer.name(DESCRIPTION_FIELD).value(description);
224 }
225 if (attributes != null && attributes.size() > 0) {
226 writer.name(ATTRIBUTES_FIELD).beginObject();
227 for (Map.Entry<String, String> attribute : attributes.entrySet()) {
228 writer.name(attribute.getKey()).value(attribute.getValue());
229 }
230 writer.endObject();
231 }
232 writer.name(VERSIONS_FIELD).value(versions);
233 writer.endObject();
234 writer.flush();
235 } finally {
236 writer.close();
237 }
238 return buffer.toByteArray();
239 }
240
241 /**
242 * Deserialize a new metadata object from a set of bytes.
243 * @param bytes the serialized metadata
244 * @throws IOException
245 */
246 protected Metadata(byte[] bytes) throws IOException {
247 String cipher = null;
248 int bitLength = 0;
249 Date created = null;
250 int versions = 0;
251 String description = null;
252 Map<String, String> attributes = null;
253 JsonReader reader = new JsonReader(new InputStreamReader
254 (new ByteArrayInputStream(bytes)));
255 try {
256 reader.beginObject();
257 while (reader.hasNext()) {
258 String field = reader.nextName();
259 if (CIPHER_FIELD.equals(field)) {
260 cipher = reader.nextString();
261 } else if (BIT_LENGTH_FIELD.equals(field)) {
262 bitLength = reader.nextInt();
263 } else if (CREATED_FIELD.equals(field)) {
264 created = new Date(reader.nextLong());
265 } else if (VERSIONS_FIELD.equals(field)) {
266 versions = reader.nextInt();
267 } else if (DESCRIPTION_FIELD.equals(field)) {
268 description = reader.nextString();
269 } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
270 reader.beginObject();
271 attributes = new HashMap<String, String>();
272 while (reader.hasNext()) {
273 attributes.put(reader.nextName(), reader.nextString());
274 }
275 reader.endObject();
276 }
277 }
278 reader.endObject();
279 } finally {
280 reader.close();
281 }
282 this.cipher = cipher;
283 this.bitLength = bitLength;
284 this.created = created;
285 this.description = description;
286 this.attributes = attributes;
287 this.versions = versions;
288 }
289 }
290
291 /**
292 * Options when creating key objects.
293 */
294 public static class Options {
295 private String cipher;
296 private int bitLength;
297 private String description;
298 private Map<String, String> attributes;
299
300 public Options(Configuration conf) {
301 cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
302 bitLength = conf.getInt(DEFAULT_BITLENGTH_NAME, DEFAULT_BITLENGTH);
303 }
304
305 public Options setCipher(String cipher) {
306 this.cipher = cipher;
307 return this;
308 }
309
310 public Options setBitLength(int bitLength) {
311 this.bitLength = bitLength;
312 return this;
313 }
314
315 public Options setDescription(String description) {
316 this.description = description;
317 return this;
318 }
319
320 public Options setAttributes(Map<String, String> attributes) {
321 if (attributes != null) {
322 if (attributes.containsKey(null)) {
323 throw new IllegalArgumentException("attributes cannot have a NULL key");
324 }
325 this.attributes = new HashMap<String, String>(attributes);
326 }
327 return this;
328 }
329
330 public String getCipher() {
331 return cipher;
332 }
333
334 public int getBitLength() {
335 return bitLength;
336 }
337
338 public String getDescription() {
339 return description;
340 }
341
342 @SuppressWarnings("unchecked")
343 public Map<String, String> getAttributes() {
344 return (attributes == null) ? Collections.EMPTY_MAP : attributes;
345 }
346
347 @Override
348 public String toString() {
349 return "Options{" +
350 "cipher='" + cipher + '\'' +
351 ", bitLength=" + bitLength +
352 ", description='" + description + '\'' +
353 ", attributes=" + attributes +
354 '}';
355 }
356 }
357
358 /**
359 * Constructor.
360 *
361 * @param conf configuration for the provider
362 */
363 public KeyProvider(Configuration conf) {
364 this.conf = new Configuration(conf);
365 }
366
367 /**
368 * Return the provider configuration.
369 *
370 * @return the provider configuration
371 */
372 public Configuration getConf() {
373 return conf;
374 }
375
376 /**
377 * A helper function to create an options object.
378 * @param conf the configuration to use
379 * @return a new options object
380 */
381 public static Options options(Configuration conf) {
382 return new Options(conf);
383 }
384
385 /**
386 * Indicates whether this provider represents a store
387 * that is intended for transient use - such as the UserProvider
388 * is. These providers are generally used to provide access to
389 * keying material rather than for long term storage.
390 * @return true if transient, false otherwise
391 */
392 public boolean isTransient() {
393 return false;
394 }
395
396 /**
397 * Get the key material for a specific version of the key. This method is used
398 * when decrypting data.
399 * @param versionName the name of a specific version of the key
400 * @return the key material
401 * @throws IOException
402 */
403 public abstract KeyVersion getKeyVersion(String versionName
404 ) throws IOException;
405
406 /**
407 * Get the key names for all keys.
408 * @return the list of key names
409 * @throws IOException
410 */
411 public abstract List<String> getKeys() throws IOException;
412
413 /**
414 * Get key metadata in bulk.
415 * @param names the names of the keys to get
416 * @throws IOException
417 */
418 public Metadata[] getKeysMetadata(String... names) throws IOException {
419 Metadata[] result = new Metadata[names.length];
420 for (int i=0; i < names.length; ++i) {
421 result[i] = getMetadata(names[i]);
422 }
423 return result;
424 }
425
426 /**
427 * Get the key material for all versions of a specific key name.
428 * @return the list of key material
429 * @throws IOException
430 */
431 public abstract List<KeyVersion> getKeyVersions(String name) throws IOException;
432
433 /**
434 * Get the current version of the key, which should be used for encrypting new
435 * data.
436 * @param name the base name of the key
437 * @return the version name of the current version of the key or null if the
438 * key version doesn't exist
439 * @throws IOException
440 */
441 public KeyVersion getCurrentKey(String name) throws IOException {
442 Metadata meta = getMetadata(name);
443 if (meta == null) {
444 return null;
445 }
446 return getKeyVersion(buildVersionName(name, meta.getVersions() - 1));
447 }
448
449 /**
450 * Get metadata about the key.
451 * @param name the basename of the key
452 * @return the key's metadata or null if the key doesn't exist
453 * @throws IOException
454 */
455 public abstract Metadata getMetadata(String name) throws IOException;
456
457 /**
458 * Create a new key. The given key must not already exist.
459 * @param name the base name of the key
460 * @param material the key material for the first version of the key.
461 * @param options the options for the new key.
462 * @return the version name of the first version of the key.
463 * @throws IOException
464 */
465 public abstract KeyVersion createKey(String name, byte[] material,
466 Options options) throws IOException;
467
468 /**
469 * Get the algorithm from the cipher.
470 *
471 * @return the algorithm name
472 */
473 private String getAlgorithm(String cipher) {
474 int slash = cipher.indexOf('/');
475 if (slash == -1) {
476 return cipher;
477 } else {
478 return cipher.substring(0, slash);
479 }
480 }
481
482 /**
483 * Generates a key material.
484 *
485 * @param size length of the key.
486 * @param algorithm algorithm to use for generating the key.
487 * @return the generated key.
488 * @throws NoSuchAlgorithmException
489 */
490 protected byte[] generateKey(int size, String algorithm)
491 throws NoSuchAlgorithmException {
492 algorithm = getAlgorithm(algorithm);
493 KeyGenerator keyGenerator = KeyGenerator.getInstance(algorithm);
494 keyGenerator.init(size);
495 byte[] key = keyGenerator.generateKey().getEncoded();
496 return key;
497 }
498
499 /**
500 * Create a new key generating the material for it.
501 * The given key must not already exist.
502 * <p/>
503 * This implementation generates the key material and calls the
504 * {@link #createKey(String, byte[], Options)} method.
505 *
506 * @param name the base name of the key
507 * @param options the options for the new key.
508 * @return the version name of the first version of the key.
509 * @throws IOException
510 * @throws NoSuchAlgorithmException
511 */
512 public KeyVersion createKey(String name, Options options)
513 throws NoSuchAlgorithmException, IOException {
514 byte[] material = generateKey(options.getBitLength(), options.getCipher());
515 return createKey(name, material, options);
516 }
517
518 /**
519 * Delete the given key.
520 * @param name the name of the key to delete
521 * @throws IOException
522 */
523 public abstract void deleteKey(String name) throws IOException;
524
525 /**
526 * Roll a new version of the given key.
527 * @param name the basename of the key
528 * @param material the new key material
529 * @return the name of the new version of the key
530 * @throws IOException
531 */
532 public abstract KeyVersion rollNewVersion(String name,
533 byte[] material
534 ) throws IOException;
535
536 /**
537 * Can be used by implementing classes to close any resources
538 * that require closing
539 */
540 public void close() throws IOException {
541 // NOP
542 }
543
544 /**
545 * Roll a new version of the given key generating the material for it.
546 * <p/>
547 * This implementation generates the key material and calls the
548 * {@link #rollNewVersion(String, byte[])} method.
549 *
550 * @param name the basename of the key
551 * @return the name of the new version of the key
552 * @throws IOException
553 */
554 public KeyVersion rollNewVersion(String name) throws NoSuchAlgorithmException,
555 IOException {
556 Metadata meta = getMetadata(name);
557 byte[] material = generateKey(meta.getBitLength(), meta.getCipher());
558 return rollNewVersion(name, material);
559 }
560
561 /**
562 * Ensures that any changes to the keys are written to persistent store.
563 * @throws IOException
564 */
565 public abstract void flush() throws IOException;
566
567 /**
568 * Split the versionName in to a base name. Converts "/aaa/bbb/3" to
569 * "/aaa/bbb".
570 * @param versionName the version name to split
571 * @return the base name of the key
572 * @throws IOException
573 */
574 public static String getBaseName(String versionName) throws IOException {
575 int div = versionName.lastIndexOf('@');
576 if (div == -1) {
577 throw new IOException("No version in key path " + versionName);
578 }
579 return versionName.substring(0, div);
580 }
581
582 /**
583 * Build a version string from a basename and version number. Converts
584 * "/aaa/bbb" and 3 to "/aaa/bbb@3".
585 * @param name the basename of the key
586 * @param version the version of the key
587 * @return the versionName of the key.
588 */
589 protected static String buildVersionName(String name, int version) {
590 return name + "@" + version;
591 }
592
593 /**
594 * Find the provider with the given key.
595 * @param providerList the list of providers
596 * @param keyName the key name we are looking for
597 * @return the KeyProvider that has the key
598 */
599 public static KeyProvider findProvider(List<KeyProvider> providerList,
600 String keyName) throws IOException {
601 for(KeyProvider provider: providerList) {
602 if (provider.getMetadata(keyName) != null) {
603 return provider;
604 }
605 }
606 throw new IOException("Can't find KeyProvider for key " + keyName);
607 }
608 }