#ifndef VIENNACL_LINALG_KERNELS_MATRIX_COL_SOURCE_HPP_
#define VIENNACL_LINALG_KERNELS_MATRIX_COL_SOURCE_HPP_
//Automatically generated file from auxiliary-directory, do not edit manually!
/** @file matrix_col_source.h
 *  @brief OpenCL kernel source file, generated automatically. */
namespace viennacl
{
 namespace linalg
 {
  namespace kernels
  {
const char * const matrix_col_align1_ambm_m_cpu_gpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_m_cpu_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          float fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          __global const float * fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3[0];\n"
"  if ((options3 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options3 >> 2); ++i)\n"
"      beta += fac3[i];\n"
"  }\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"   += B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_m_cpu_gpu

const char * const matrix_col_align1_ambm_m_cpu_cpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_m_cpu_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          float fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          float fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3;\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"   += B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_m_cpu_cpu

const char * const matrix_col_align1_ambm_m_gpu_cpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_m_gpu_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          float fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3;\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"   += B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_m_gpu_cpu

const char * const matrix_col_align1_lu_factorize = 
"__kernel void lu_factorize(\n"
"          __global float * matrix,\n"
"          unsigned int matrix_rows,\n"
"          unsigned int matrix_cols,\n"
"          unsigned int matrix_internal_rows,\n"
"          unsigned int matrix_internal_cols) \n"
"{ \n"
"  float temp;\n"
"  for (unsigned int i=1; i<matrix_rows; ++i)\n"
"  {\n"
"    for (unsigned int k=0; k<i; ++k)\n"
"    {\n"
"      if (get_global_id(0) == 0)\n"
"        matrix[i + k*matrix_internal_rows] /= matrix[k + k*matrix_internal_rows];\n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"      temp = matrix[i + k*matrix_internal_rows];\n"
"      \n"
"      //parallel subtraction:\n"
"      for (unsigned int j=k+1 + get_global_id(0); j<matrix_cols; j += get_global_size(0))\n"
"        matrix[i + j*matrix_internal_rows] -= temp * matrix[k + j*matrix_internal_rows];\n"
"    }\n"
"  }\n"
"} \n"
; //matrix_col_align1_lu_factorize

const char * const matrix_col_align1_fft_reorder = 
"/*\n"
"* Performs reordering of input data in bit-reversal order\n"
"* Probably it's better to do in host side,\n"
"*/\n"
"unsigned int get_reorder_num_2(unsigned int v, unsigned int bit_size) {\n"
"    v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1);\n"
"    v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2);\n"
"    v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4);\n"
"    v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8);\n"
"    v = (v >> 16) | (v << 16);\n"
"    v = v >> (32 - bit_size);\n"
"    return v;\n"
"}\n"
"__kernel void fft_reorder(__global float2* input,\n"
"                          unsigned int bit_size,\n"
"                          unsigned int size,\n"
"                          unsigned int stride,\n"
"                          int batch_num) {\n"
"    unsigned int glb_id = get_global_id(0);\n"
"    unsigned int glb_sz = get_global_size(0);\n"
"	\n"
"    for(unsigned int batch_id = 0; batch_id < batch_num; batch_id++) {\n"
"        for(unsigned int i = glb_id; i < size; i += glb_sz) {\n"
"            unsigned int v = get_reorder_num_2(i, bit_size);\n"
"            if(i < v) {\n"
"                float2 tmp = input[i * stride + batch_id]; // index\n"
"                input[i * stride + batch_id] = input[v * stride + batch_id]; //index\n"
"                input[v * stride + batch_id] = tmp; //index\n"
"            }\n"
"        }\n"
"    }\n"
"}\n"
; //matrix_col_align1_fft_reorder

const char * const matrix_col_align1_fft_radix2 = 
"__kernel void fft_radix2(__global float2* input,\n"
"                         unsigned int s,\n"
"                         unsigned int bit_size,\n"
"                         unsigned int size,\n"
"                         unsigned int stride,\n"
"                         unsigned int batch_num,\n"
"                         float sign) {\n"
"    unsigned int ss = 1 << s;\n"
"    unsigned int half_size = size >> 1;\n"
"    float cs, sn;\n"
"    const float NUM_PI = 3.14159265358979323846;\n"
"    unsigned int glb_id = get_global_id(0);\n"
"    unsigned int glb_sz = get_global_size(0);\n"
"	\n"
"    for(unsigned int batch_id = 0; batch_id < batch_num; batch_id++) {\n"
"        for(unsigned int tid = glb_id; tid < half_size; tid += glb_sz) {\n"
"            unsigned int group = (tid & (ss - 1));\n"
"            unsigned int pos = ((tid >> s) << (s + 1)) + group;\n"
"            unsigned int offset = pos * stride + batch_id;\n"
"            float2 in1 = input[offset];//index\n"
"            float2 in2 = input[offset + ss * stride];//index\n"
"            float arg = group * sign * NUM_PI / ss;\n"
"            sn = sincos(arg, &cs);\n"
"            float2 ex = (float2)(cs, sn);\n"
"            float2 tmp = (float2)(in2.x * ex.x - in2.y * ex.y, in2.x * ex.y + in2.y * ex.x);\n"
"            input[offset + ss * stride] = in1 - tmp;//index\n"
"            input[offset] = in1 + tmp;//index\n"
"        }\n"
"    }\n"
"}\n"
; //matrix_col_align1_fft_radix2

const char * const matrix_col_align1_ambm_gpu_gpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_gpu_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          __global const float * fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3[0];\n"
"  if ((options3 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options3 >> 2); ++i)\n"
"      beta += fac3[i];\n"
"  }\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"    = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_gpu_gpu

const char * const matrix_col_align1_am_gpu = 
"// generic kernel for the matrix operation A = alpha * B, where A, B are not necessarily distinct matrices\n"
"__kernel void am_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2)\n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha;\n"
"}\n"
; //matrix_col_align1_am_gpu

const char * const matrix_col_align1_fft_direct = 
"// Direct FFT computation (quadratic complexity - use for reference only)\n"
"__kernel void fft_direct(__global float2* input,\n"
"                         __global float2* output,\n"
"                         unsigned int size,\n"
"                         unsigned int stride,\n"
"                         unsigned int batch_num,\n"
"                         float sign) {\n"
"                         \n"
"    const float NUM_PI = 3.14159265358979323846;\n"
"    \n"
"    for(unsigned int batch_id = 0; batch_id < batch_num; batch_id++) {\n"
"        for(unsigned int k = get_global_id(0); k < size; k += get_global_size(0)) {\n"
"            float2 f = 0.0f;\n"
"            for(unsigned int n = 0; n < size; n++) {\n"
"                float2 in = input[n * stride + batch_id]; //input index here\n"
"                float sn, cs;\n"
"                float arg = sign * 2 * NUM_PI * k / size * n;\n"
"                sn = sincos(arg, &cs);\n"
"                float2 ex = (float2)(cs, sn);\n"
"                f = f + (float2)(in.x * ex.x - in.y * ex.y, in.x * ex.y + in.y * ex.x);\n"
"            }\n"
"            output[k * stride + batch_id] = f;// output index here\n"
"        }\n"
"    }\n"
"}\n"
; //matrix_col_align1_fft_direct

const char * const matrix_col_align1_trans_vec_mul = 
"__kernel void trans_vec_mul(\n"
"          __global const float * A,\n"
"          unsigned int A_row_start,\n"
"          unsigned int A_col_start,\n"
"          unsigned int A_row_inc,\n"
"          unsigned int A_col_inc,\n"
"          unsigned int A_row_size,\n"
"          unsigned int A_col_size,\n"
"          unsigned int A_internal_rows,\n"
"          unsigned int A_internal_cols,\n"
"          __global const float * v,\n"
"          unsigned int v_start,\n"
"          unsigned int v_inc,\n"
"          unsigned int v_size,\n"
"          __global float * result,\n"
"          unsigned int result_start,\n"
"          unsigned int result_inc,\n"
"          unsigned int result_size,\n"
"          __local float * work) \n"
"{ \n"
"  unsigned int row_gid = get_global_id(0) / get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int lid = get_local_id(0);\n"
"  \n"
"  for (unsigned int row = row_gid; row < A_col_size; row += get_num_groups(0))\n"
"  {\n"
"    float dot_prod = 0;\n"
"    for (unsigned int col = col_gid; col < A_row_size; col+=get_local_size(0))\n"
"      dot_prod += A[(row * A_col_inc + A_col_start) * A_internal_rows + col * A_row_inc + A_row_start] * v[v_start + v_inc * col];\n"
"    work[lid] = dot_prod;\n"
"    \n"
"    for(unsigned int stride=get_local_size(0)/2 ; stride>0 ; stride>>=1){\n"
"      barrier(CLK_LOCAL_MEM_FENCE);\n"
"      if(lid < stride)\n"
"        work[lid] += work[lid+stride];\n"
"    }\n"
"    \n"
"    if(lid == 0)\n"
"      result[row * result_inc + result_start] = work[0];\n"
"  }\n"
"}\n"
; //matrix_col_align1_trans_vec_mul

const char * const matrix_col_align1_am_cpu = 
"// generic kernel for the matrix operation A = alpha * B, where A, B are not necessarily distinct matrices\n"
"__kernel void am_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          float fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2)\n"
"{ \n"
"  float alpha = fac2;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha;\n"
"}\n"
; //matrix_col_align1_am_cpu

const char * const matrix_col_align1_scaled_rank1_update_gpu = 
"__kernel void scaled_rank1_update_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          \n"
"          __global const float * vec1,\n"
"          unsigned int start1,\n"
"          unsigned int inc1,          \n"
"          unsigned int size1,\n"
"          __global const float * vec2,\n"
"          unsigned int start2,\n"
"          unsigned int inc2,          \n"
"          unsigned int size2) \n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  unsigned int row_gid = get_global_id(0) / get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) % get_local_size(0);\n"
"  \n"
"  for (unsigned int row = row_gid; row < A_size1; row += get_num_groups(0))\n"
"  {\n"
"    float tmp = alpha * vec1[row * inc1 + start1];\n"
"    for (unsigned int col = col_gid; col < A_size2; col += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] += tmp * vec2[col * inc2 + start2];\n"
"  }\n"
"}\n"
; //matrix_col_align1_scaled_rank1_update_gpu

const char * const matrix_col_align1_assign_cpu = 
"__kernel void assign_cpu(\n"
"          __global float * A,\n"
"          unsigned int start1,          unsigned int start2,\n"
"          unsigned int inc1,            unsigned int inc2,\n"
"          unsigned int size1,           unsigned int size2,\n"
"          unsigned int internal_size1,  unsigned int internal_size2,\n"
"          float alpha) \n"
"{ \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < size1; row += get_local_size(0))\n"
"      A[(row * inc1 + start1) + (col * inc2 + start2) * internal_size1] = alpha;\n"
"}\n"
; //matrix_col_align1_assign_cpu

const char * const matrix_col_align1_fft_radix2_local = 
"unsigned int get_reorder_num(unsigned int v, unsigned int bit_size) {\n"
"    v = ((v >> 1) & 0x55555555) | ((v & 0x55555555) << 1);\n"
"    v = ((v >> 2) & 0x33333333) | ((v & 0x33333333) << 2);\n"
"    v = ((v >> 4) & 0x0F0F0F0F) | ((v & 0x0F0F0F0F) << 4);\n"
"    v = ((v >> 8) & 0x00FF00FF) | ((v & 0x00FF00FF) << 8);\n"
"    v = (v >> 16) | (v << 16);\n"
"    v = v >> (32 - bit_size);\n"
"    return v;\n"
"}\n"
"__kernel void fft_radix2_local(__global float2* input,\n"
"                                __local float2* lcl_input,\n"
"                                unsigned int bit_size,\n"
"                                unsigned int size,\n"
"                                unsigned int stride,\n"
"                                unsigned int batch_num,\n"
"                                float sign) {\n"
"    unsigned int grp_id = get_group_id(0);\n"
"    unsigned int grp_num = get_num_groups(0);\n"
"    unsigned int lcl_sz = get_local_size(0);\n"
"    unsigned int lcl_id = get_local_id(0);\n"
"    const float NUM_PI = 3.14159265358979323846;\n"
"    for(unsigned int batch_id = grp_id; batch_id < batch_num; batch_id += grp_num) {\n"
"        //unsigned int base_offset = stride * batch_id;\n"
"        //copy chunk of global memory to local\n"
"        for(unsigned int p = lcl_id; p < size; p += lcl_sz) {\n"
"            unsigned int v = get_reorder_num(p, bit_size);\n"
"            lcl_input[v] = input[p * stride + batch_id];//index\n"
"        }\n"
"        barrier(CLK_LOCAL_MEM_FENCE);\n"
"        //performs Cooley-Tukey FFT on local array\n"
"        for(unsigned int s = 0; s < bit_size; s++) {\n"
"            unsigned int ss = 1 << s;\n"
"            float cs, sn;\n"
"            for(unsigned int tid = lcl_id; tid < size; tid += lcl_sz) {\n"
"                unsigned int group = (tid & (ss - 1));\n"
"                unsigned int pos = ((tid >> s) << (s + 1)) + group;\n"
"                float2 in1 = lcl_input[pos];\n"
"                float2 in2 = lcl_input[pos + ss];\n"
"                float arg = group * sign * NUM_PI / ss;\n"
"                sn = sincos(arg, &cs);\n"
"                float2 ex = (float2)(cs, sn);\n"
"                float2 tmp = (float2)(in2.x * ex.x - in2.y * ex.y, in2.x * ex.y + in2.y * ex.x);\n"
"                lcl_input[pos + ss] = in1 - tmp;\n"
"                lcl_input[pos] = in1 + tmp;\n"
"            }\n"
"            barrier(CLK_LOCAL_MEM_FENCE);\n"
"        }\n"
"        //copy local array back to global memory\n"
"        for(unsigned int p = lcl_id; p < size; p += lcl_sz) {\n"
"            input[p * stride + batch_id] = lcl_input[p];//index\n"
"        }\n"
"    }\n"
"}\n"
; //matrix_col_align1_fft_radix2_local

const char * const matrix_col_align1_triangular_substitute_inplace = 
"__kernel void triangular_substitute_inplace(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          __global float * v,\n"
"          unsigned int v_start,\n"
"          unsigned int v_inc,\n"
"          unsigned int v_size,\n"
"          unsigned int options)\n"
"{\n"
"  float temp;\n"
"  unsigned int unit_diagonal_flag  = (options & (1 << 0));\n"
"  unsigned int transposed_access_A = (options & (1 << 1));\n"
"  unsigned int is_lower_solve      = (options & (1 << 2));\n"
"  unsigned int row;\n"
"  for (unsigned int rows_processed = 0; rows_processed < A_size1; ++rows_processed)    //Note: A required to be square\n"
"  {\n"
"    row = is_lower_solve ? rows_processed : ((A_size1 - rows_processed) - 1);\n"
"    if (!unit_diagonal_flag)\n"
"    {\n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"      if (get_global_id(0) == 0)\n"
"        v[row * v_inc + v_start] /= A[(row * A_inc1 + A_start1) + (row * A_inc2 + A_start2) * A_internal_size1];\n"
"    }\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    temp = v[row * v_inc + v_start];\n"
"    for (int elim = (is_lower_solve ? (row + get_global_id(0) + 1) : get_global_id(0));\n"
"             elim < (is_lower_solve ? A_size1 : row);\n"
"             elim += get_global_size(0))\n"
"      v[elim * v_inc + v_start] -= temp * A[transposed_access_A ? ((row  * A_inc1 + A_start1) + (elim * A_inc2 + A_start2) * A_internal_size1) \n"
"                                                                : ((elim * A_inc1 + A_start1) + (row  * A_inc2 + A_start2) * A_internal_size1)];\n"
"  }\n"
"}\n"
; //matrix_col_align1_triangular_substitute_inplace

const char * const matrix_col_align1_ambm_m_gpu_gpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_m_gpu_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          __global const float * fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3[0];\n"
"  if ((options3 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options3 >> 2); ++i)\n"
"      beta += fac3[i];\n"
"  }\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"   += B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_m_gpu_gpu

const char * const matrix_col_align1_vec_mul = 
"__kernel void vec_mul(\n"
"          __global const float * A,\n"
"          unsigned int A_row_start,\n"
"          unsigned int A_col_start,\n"
"          unsigned int A_row_inc,\n"
"          unsigned int A_col_inc,\n"
"          unsigned int A_row_size,\n"
"          unsigned int A_col_size,\n"
"          unsigned int A_internal_rows,\n"
"          unsigned int A_internal_cols,\n"
"          __global const float * v,\n"
"          unsigned int v_start,\n"
"          unsigned int v_inc,\n"
"          unsigned int v_size,\n"
"          __global float * result,\n"
"          unsigned int result_start,\n"
"          unsigned int result_inc,\n"
"          unsigned int result_size,\n"
"          __local float * work) \n"
"{ \n"
"  for (unsigned int row = get_global_id(0); row < A_row_size; row += get_global_size(0))\n"
"  {\n"
"    float dot_prod = 0;\n"
"    for (unsigned int col = 0; col < A_col_size; ++col)\n"
"      dot_prod += A[(row * A_row_inc + A_row_start) + (col * A_col_inc + A_col_start) * A_internal_rows] * v[v_start + v_inc * col];\n"
"    result[row * result_inc + result_start] = dot_prod;\n"
"  }\n"
"}\n"
; //matrix_col_align1_vec_mul

const char * const matrix_col_align1_ambm_gpu_cpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_gpu_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          __global const float * fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          float fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2[0];\n"
"  if ((options2 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options2 >> 2); ++i)\n"
"      alpha += fac2[i];\n"
"  }\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3;\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"    = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_gpu_cpu

const char * const matrix_col_align1_ambm_cpu_cpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_cpu_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          float fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          float fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3;\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"    = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_cpu_cpu

const char * const matrix_col_align1_ambm_cpu_gpu = 
"// generic kernel for the matrix operation A = alpha * B + beta * C, where A, B, C are not necessarily distinct matrices\n"
"__kernel void ambm_cpu_gpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          \n"
"          float fac2,\n"
"          unsigned int options2,\n"
"          __global const float * B,\n"
"          unsigned int B_start1, unsigned int B_start2,\n"
"          unsigned int B_inc1,   unsigned int B_inc2,\n"
"          unsigned int B_internal_size1,  unsigned int B_internal_size2,\n"
"          \n"
"          __global const float * fac3,\n"
"          unsigned int options3,\n"
"          __global const float * C,\n"
"          unsigned int C_start1, unsigned int C_start2,\n"
"          unsigned int C_inc1,   unsigned int C_inc2,\n"
"          unsigned int C_internal_size1,  unsigned int C_internal_size2)\n"
"{ \n"
"  float alpha = fac2;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  float beta = fac3[0];\n"
"  if ((options3 >> 2) > 1)\n"
"  {\n"
"    for (unsigned int i=1; i<(options3 >> 2); ++i)\n"
"      beta += fac3[i];\n"
"  }\n"
"  if (options3 & (1 << 0))\n"
"    beta = -beta;\n"
"  if (options3 & (1 << 1))\n"
"    beta = ((float)(1)) / beta;\n"
"    \n"
"  unsigned int row_gid = get_global_id(0) % get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) / get_local_size(0);\n"
"  \n"
"  for (unsigned int col = col_gid; col < A_size2; col += get_num_groups(0))\n"
"    for (unsigned int row = row_gid; row < A_size1; row += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] \n"
"    = B[(row * B_inc1 + B_start1) + (col * B_inc2 + B_start2) * B_internal_size1] * alpha\n"
"    + C[(row * C_inc1 + C_start1) + (col * C_inc2 + C_start2) * C_internal_size1] * beta;\n"
"}\n"
; //matrix_col_align1_ambm_cpu_gpu

const char * const matrix_col_align1_scaled_rank1_update_cpu = 
"__kernel void scaled_rank1_update_cpu(\n"
"          __global float * A,\n"
"          unsigned int A_start1, unsigned int A_start2,\n"
"          unsigned int A_inc1,   unsigned int A_inc2,\n"
"          unsigned int A_size1,  unsigned int A_size2,\n"
"          unsigned int A_internal_size1,  unsigned int A_internal_size2,\n"
"          float val,\n"
"          unsigned int options2,\n"
"          \n"
"          __global const float * vec1,\n"
"          unsigned int start1,\n"
"          unsigned int inc1,          \n"
"          unsigned int size1,\n"
"          __global const float * vec2,\n"
"          unsigned int start2,\n"
"          unsigned int inc2,          \n"
"          unsigned int size2) \n"
"{ \n"
"  float alpha = val;\n"
"  if (options2 & (1 << 0))\n"
"    alpha = -alpha;\n"
"  if (options2 & (1 << 1))\n"
"    alpha = ((float)(1)) / alpha;\n"
"  unsigned int row_gid = get_global_id(0) / get_local_size(0);\n"
"  unsigned int col_gid = get_global_id(0) % get_local_size(0);\n"
"  \n"
"  for (unsigned int row = row_gid; row < A_size1; row += get_num_groups(0))\n"
"  {\n"
"    float tmp = alpha * vec1[row * inc1 + start1];\n"
"    for (unsigned int col = col_gid; col < A_size2; col += get_local_size(0))\n"
"      A[(row * A_inc1 + A_start1) + (col * A_inc2 + A_start2) * A_internal_size1] += tmp * vec2[col * inc2 + start2];\n"
"  }\n"
"}\n"
; //matrix_col_align1_scaled_rank1_update_cpu

const char * const matrix_col_align1_diagonal_assign_cpu = 
"__kernel void diagonal_assign_cpu(\n"
"          __global float * A,\n"
"          unsigned int start1,          unsigned int start2,\n"
"          unsigned int inc1,            unsigned int inc2,\n"
"          unsigned int size1,           unsigned int size2,\n"
"          unsigned int internal_size1,  unsigned int internal_size2,\n"
"          float alpha) \n"
"{ \n"
"  for (unsigned int col = get_global_id(0); col < size2; col += get_global_size(0))\n"
"    A[(col * inc1 + start1) + (col * inc2 + start2) * internal_size1] = alpha;\n"
"}\n"
; //matrix_col_align1_diagonal_assign_cpu

  }  //namespace kernels
 }  //namespace linalg
}  //namespace viennacl
#endif

