InternalError : Blas GEMM launch failed

# 플레이스홀더와 변수의 개념을 익혀봅니다
import tensorflow as tf

# tf.placeholder: 계산을 실행할 때 입력값을 받는 변수로 사용합니다.
# None 은 크기가 정해지지 않았음을 의미합니다.
X = tf.placeholder(tf.float32, [None, 3])
print(X)

# X 플레이스홀더에 넣을 값 입니다.
# 플레이스홀더에서 설정한 것 처럼, 두번째 차원의 요소의 갯수는 3개 입니다.
x_data = [[1, 2, 3], [4, 5, 6]]

# tf.Variable: 그래프를 계산하면서 최적화 할 변수들입니다. 이 값이 바로 신경망을 좌우하는 값들입니다.
# tf.random_normal: 각 변수들의 초기값을 정규분포 랜덤 값으로 초기화합니다.
W = tf.Variable(tf.random_normal([3, 2]))
b = tf.Variable(tf.random_normal([2, 1]))

# 입력값과 변수들을 계산할 수식을 작성합니다.
# tf.matmul 처럼 mat* 로 되어 있는 함수로 행렬 계산을 수행합니다.
expr = tf.matmul(X, W) + b

sess = tf.Session()
# 위에서 설정한 Variable 들의 값들을 초기화 하기 위해
# 처음에 tf.global_variables_initializer 를 한 번 실행해야 합니다.
sess.run(tf.global_variables_initializer())

print("=== x_data ===")
print(x_data)
print("=== W ===")
print(sess.run(W))
print("=== b ===")
print(sess.run(b))
print("=== expr ===")
# expr 수식에는 X 라는 입력값이 필요합니다.
# 따라서 expr 실행시에는 이 변수에 대한 실제 입력값을 다음처럼 넣어줘야합니다.
print(sess.run(expr, feed_dict={X: x_data}))

sess.close()

C:\Users\Heeseok\PycharmProjects\tensorflow\venv\Scripts\python.exe "C:/Users/Heeseok/PycharmProjects/tensorflow/02 - Variable.py"

Tensor("Placeholder:0", shape=(?, 3), dtype=float32)

2018-01-01 18:12:21.673288: I C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\core\platform\cpu_feature_guard.cc:137] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX AVX2

2018-01-01 18:12:22.066072: I C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\core\common_runtime\gpu\gpu_device.cc:1030] Found device 0 with properties: 

name: GeForce GTX 960M major: 5 minor: 0 memoryClockRate(GHz): 1.176

pciBusID: 0000:01:00.0

totalMemory: 4.00GiB freeMemory: 3.35GiB

2018-01-01 18:12:22.066414: I C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\core\common_runtime\gpu\gpu_device.cc:1120] Creating TensorFlow device (/device:GPU:0) -> (device: 0, name: GeForce GTX 960M, pci bus id: 0000:01:00.0, compute capability: 5.0)

=== x_data ===

[[1, 2, 3], [4, 5, 6]]

=== W ===

[[-0.56218028 -1.56666601]

 [ 0.46365535  0.02119004]

 [-0.88355196 -0.47984856]]

=== b ===

[[-0.55556625]

 [ 2.29203272]]

=== expr ===

2018-01-01 18:12:23.336437: E C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\stream_executor\cuda\cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_ALLOC_FAILED

2018-01-01 18:12:23.337101: E C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\stream_executor\cuda\cuda_blas.cc:366] failed to create cublas handle: CUBLAS_STATUS_ALLOC_FAILED

2018-01-01 18:12:23.337285: W C:\tf_jenkins\home\workspace\rel-win\M\windows-gpu\PY\36\tensorflow\stream_executor\stream.cc:1901] attempting to perform BLAS operation using StreamExecutor without BLAS support

Traceback (most recent call last):

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1323, in _do_call

    return fn(*args)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1302, in _run_fn

    status, run_metadata)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\errors_impl.py", line 473, in __exit__

    c_api.TF_GetCode(self.status.status))

tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(2, 3), b.shape=(3, 2), m=2, n=2, k=3

[[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](_arg_Placeholder_0_0/_5, Variable/read)]]

[[Node: add/_7 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_12_add", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):

  File "C:/Users/Heeseok/PycharmProjects/tensorflow/02 - Variable.py", line 36, in <module>

    print(sess.run(expr, feed_dict={X: x_data}))

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 889, in run

    run_metadata_ptr)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1120, in _run

    feed_dict_tensor, options, run_metadata)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1317, in _do_run

    options, run_metadata)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\client\session.py", line 1336, in _do_call

    raise type(e)(node_def, op, message)

tensorflow.python.framework.errors_impl.InternalError: Blas GEMM launch failed : a.shape=(2, 3), b.shape=(3, 2), m=2, n=2, k=3

[[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](_arg_Placeholder_0_0/_5, Variable/read)]]

[[Node: add/_7 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_12_add", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

Caused by op 'MatMul', defined at:

  File "C:/Users/Heeseok/PycharmProjects/tensorflow/02 - Variable.py", line 20, in <module>

    expr = tf.matmul(X, W) + b

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\math_ops.py", line 1891, in matmul

    a, b, transpose_a=transpose_a, transpose_b=transpose_b, name=name)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 2436, in _mat_mul

    name=name)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper

    op_def=op_def)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 2956, in create_op

    op_def=op_def)

  File "C:\Program Files\Python36\lib\site-packages\tensorflow\python\framework\ops.py", line 1470, in __init__

    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access

InternalError (see above for traceback): Blas GEMM launch failed : a.shape=(2, 3), b.shape=(3, 2), m=2, n=2, k=3

[[Node: MatMul = MatMul[T=DT_FLOAT, transpose_a=false, transpose_b=false, _device="/job:localhost/replica:0/task:0/device:GPU:0"](_arg_Placeholder_0_0/_5, Variable/read)]]

[[Node: add/_7 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_12_add", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]()]]

Process finished with exit code 1

[실행환경] python 3.6.4

[OS] Windows 10

[Library] tensorflow-gpu version 1.4.0  (pip로 설치)

[참고] https://stackoverflow.com/questions/37337728/tensorflow-internalerror-blas-sgemm-launch-failed

[원인] GPU Memory를 TensorFlow process가 독점하고 있는데, 다른 TensorFlow Process가 GPU Memory에 접근하려고 할때 발생하는 에러.

[해결책] TensorFlow Session 설정에서 GPU 메모리를 공유하도록 설정해도 되겠지만, 일단 가장 간단한 방법은 jupyter notebook 을 종료하고, 다시 실행시켜서 하나의 kernel만 실행시킨다. jupyter notebook 을 실행시킨 상태에서 pycharm 에서 실행시킨다면 위와 같은 에러가 발생한다.