#ifndef VIENNACL_LINALG_KERNELS_COMPRESSED_MATRIX_SOURCE_HPP_
#define VIENNACL_LINALG_KERNELS_COMPRESSED_MATRIX_SOURCE_HPP_
//Automatically generated file from auxiliary-directory, do not edit manually!
/** @file compressed_matrix_source.h
 *  @brief OpenCL kernel source file, generated automatically. */
namespace viennacl
{
 namespace linalg
 {
  namespace kernels
  {
const char * const compressed_matrix_align8_vec_mul = 
"__kernel void vec_mul(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const uint8 * column_indices, \n"
"          __global const float8 * elements,\n"
"          __global const float * vector,  \n"
"          __global float * result,\n"
"          unsigned int size)\n"
"{ \n"
"  float dot_prod;\n"
"  unsigned int start, next_stop;\n"
"  uint8 col_idx;\n"
"  float8 tmp_vec;\n"
"  float8 tmp_entries;\n"
"  for (unsigned int row = get_global_id(0); row < size; row += get_global_size(0))\n"
"  {\n"
"    dot_prod = 0.0f;\n"
"    start = row_indices[row] / 8;\n"
"    next_stop = row_indices[row+1] / 8;\n"
"    for (unsigned int i = start; i < next_stop; ++i)\n"
"    {\n"
"      col_idx = column_indices[i];\n"
"      tmp_entries = elements[i];\n"
"      tmp_vec.s0 = vector[col_idx.s0];\n"
"      tmp_vec.s1 = vector[col_idx.s1];\n"
"      tmp_vec.s2 = vector[col_idx.s2];\n"
"      tmp_vec.s3 = vector[col_idx.s3];\n"
"      tmp_vec.s4 = vector[col_idx.s4];\n"
"      tmp_vec.s5 = vector[col_idx.s5];\n"
"      tmp_vec.s6 = vector[col_idx.s6];\n"
"      tmp_vec.s7 = vector[col_idx.s7];\n"
"      dot_prod += dot(tmp_entries.lo, tmp_vec.lo);\n"
"      dot_prod += dot(tmp_entries.hi, tmp_vec.hi);\n"
"    }\n"
"    result[row] = dot_prod;\n"
"  }\n"
"}\n"
; //compressed_matrix_align8_vec_mul

const char * const compressed_matrix_align1_unit_lu_forward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void unit_lu_forward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local  unsigned int col_index_buffer[128];\n"
"  __local  float element_buffer[128];\n"
"  __local  float vector_buffer[128];\n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int current_row = 0;\n"
"  unsigned int row_at_window_start = 0;\n"
"  float current_vector_entry = vector[0];\n"
"  unsigned int loop_end = (nnz / get_local_size(0) + 1) * get_local_size(0);\n"
"  unsigned int next_row = row_indices[1];\n"
"  \n"
"  for (unsigned int i = get_local_id(0); i < loop_end; i += get_local_size(0))\n"
"  {\n"
"    //load into shared memory (coalesced access):\n"
"    if (i < nnz)\n"
"    {\n"
"      element_buffer[get_local_id(0)] = elements[i];\n"
"      unsigned int tmp = column_indices[i];\n"
"      col_index_buffer[get_local_id(0)] = tmp;\n"
"      vector_buffer[get_local_id(0)] = vector[tmp];\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    //now a single thread does the remaining work in shared memory:\n"
"    if (get_local_id(0) == 0)\n"
"    {\n"
"      // traverse through all the loaded data:\n"
"      for (unsigned int k=0; k<get_local_size(0); ++k)\n"
"      {\n"
"        if (i+k == next_row) //current row is finished. Write back result\n"
"        {\n"
"          vector[current_row] = current_vector_entry;\n"
"          ++current_row;\n"
"          if (current_row < size) //load next row's data\n"
"          {\n"
"            next_row = row_indices[current_row+1];\n"
"            current_vector_entry = vector[current_row];\n"
"          }\n"
"        }\n"
"        \n"
"        if (current_row < size && col_index_buffer[k] < current_row) //substitute\n"
"        {\n"
"          if (col_index_buffer[k] < row_at_window_start) //use recently computed results\n"
"            current_vector_entry -= element_buffer[k] * vector_buffer[k];\n"
"          else if (col_index_buffer[k] < current_row) //use buffered data\n"
"            current_vector_entry -= element_buffer[k] * vector[col_index_buffer[k]];\n"
"        }\n"
"      } // for k\n"
"      \n"
"      row_at_window_start = current_row;\n"
"    } // if (get_local_id(0) == 0)\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"  } //for i\n"
"}\n"
; //compressed_matrix_align1_unit_lu_forward

const char * const compressed_matrix_align1_lu_backward = 
"// compute x in Ux = y for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void lu_backward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int col_index_buffer[128];\n"
"  __local float element_buffer[128];\n"
"  __local float vector_buffer[128];\n"
"          \n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int current_row = size-1;\n"
"  unsigned int row_at_window_start = size-1;\n"
"  float current_vector_entry = vector[size-1];\n"
"  float diagonal_entry = 0;\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0)) * get_local_size(0);\n"
"  unsigned int next_row = row_indices[size-1];\n"
"  \n"
"  unsigned int i = loop_end + get_local_id(0);\n"
"  while (1)\n"
"  {\n"
"    //load into shared memory (coalesced access):\n"
"    if (i < nnz)\n"
"    {\n"
"      element_buffer[get_local_id(0)] = elements[i];\n"
"      unsigned int tmp = column_indices[i];\n"
"      col_index_buffer[get_local_id(0)] = tmp;\n"
"      vector_buffer[get_local_id(0)] = vector[tmp];\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    //now a single thread does the remaining work in shared memory:\n"
"    if (get_local_id(0) == 0)\n"
"    {\n"
"      // traverse through all the loaded data from back to front:\n"
"      for (unsigned int k2=0; k2<get_local_size(0); ++k2)\n"
"      {\n"
"        unsigned int k = (get_local_size(0) - k2) - 1;\n"
"        \n"
"        if (i+k >= nnz)\n"
"          continue;\n"
"        \n"
"        if (col_index_buffer[k] > row_at_window_start) //use recently computed results\n"
"          current_vector_entry -= element_buffer[k] * vector_buffer[k];\n"
"        else if (col_index_buffer[k] > current_row) //use buffered data\n"
"          current_vector_entry -= element_buffer[k] * vector[col_index_buffer[k]];\n"
"        else if (col_index_buffer[k] == current_row)\n"
"          diagonal_entry = element_buffer[k];\n"
"        \n"
"        if (i+k == next_row) //current row is finished. Write back result\n"
"        {\n"
"          vector[current_row] = current_vector_entry / diagonal_entry;\n"
"          if (current_row > 0) //load next row's data\n"
"          {\n"
"            --current_row;\n"
"            next_row = row_indices[current_row];\n"
"            current_vector_entry = vector[current_row];\n"
"          }\n"
"        }\n"
"        \n"
"        \n"
"      } // for k\n"
"      \n"
"      row_at_window_start = current_row;\n"
"    } // if (get_local_id(0) == 0)\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    \n"
"    if (i < get_local_size(0))\n"
"      break;\n"
"    \n"
"    i -= get_local_size(0);\n"
"  } //for i\n"
"}\n"
; //compressed_matrix_align1_lu_backward

const char * const compressed_matrix_align1_trans_lu_backward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void trans_lu_backward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global const float * diagonal_entries,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int row_index_lookahead[256];\n"
"  __local unsigned int row_index_buffer[256];\n"
"  \n"
"  unsigned int row_index;\n"
"  unsigned int col_index;\n"
"  float matrix_entry;\n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int row_at_window_start = size;\n"
"  unsigned int row_at_window_end;\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0) + 1) * get_local_size(0);\n"
"  \n"
"  for (unsigned int i2 = get_local_id(0); i2 < loop_end; i2 += get_local_size(0))\n"
"  {\n"
"    unsigned int i = (nnz - i2) - 1;\n"
"    col_index    = (i2 < nnz) ? column_indices[i] : 0;\n"
"    matrix_entry = (i2 < nnz) ? elements[i]       : 0;\n"
"    row_index_lookahead[get_local_id(0)] = (row_at_window_start >= get_local_id(0)) ? row_indices[row_at_window_start - get_local_id(0)] : 0;\n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    if (i2 < nnz)\n"
"    {\n"
"      unsigned int row_index_dec = 0;\n"
"      while (row_index_lookahead[row_index_dec] > i)\n"
"        ++row_index_dec;\n"
"      row_index = row_at_window_start - row_index_dec;\n"
"      row_index_buffer[get_local_id(0)] = row_index;\n"
"    }\n"
"    else\n"
"    {\n"
"      row_index = size+1;\n"
"      row_index_buffer[get_local_id(0)] = 0;\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    row_at_window_start = row_index_buffer[0];\n"
"    row_at_window_end   = row_index_buffer[get_local_size(0) - 1];\n"
"    \n"
"    //backward elimination\n"
"    for (unsigned int row2 = 0; row2 <= (row_at_window_start - row_at_window_end); ++row2) \n"
"    { \n"
"      unsigned int row = row_at_window_start - row2;\n"
"      float result_entry = vector[row] / diagonal_entries[row];\n"
"      \n"
"      if ( (row_index == row) && (col_index < row) )\n"
"        vector[col_index] -= result_entry * matrix_entry; \n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    }\n"
"    \n"
"    row_at_window_start = row_at_window_end;\n"
"  }\n"
"  \n"
"  // final step: Divide vector by diagonal entries:\n"
"  for (unsigned int i = get_local_id(0); i < size; i += get_local_size(0))\n"
"    vector[i] /= diagonal_entries[i];\n"
"}\n"
; //compressed_matrix_align1_trans_lu_backward

const char * const compressed_matrix_align1_unit_lu_backward = 
"// compute x in Ux = y for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void unit_lu_backward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local  unsigned int col_index_buffer[128];\n"
"  __local  float element_buffer[128];\n"
"  __local  float vector_buffer[128];\n"
"  \n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int current_row = size-1;\n"
"  unsigned int row_at_window_start = size-1;\n"
"  float current_vector_entry = vector[size-1];\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0)) * get_local_size(0);\n"
"  unsigned int next_row = row_indices[size-1];\n"
"  \n"
"  unsigned int i = loop_end + get_local_id(0);\n"
"  while (1)\n"
"  {\n"
"    //load into shared memory (coalesced access):\n"
"    if (i < nnz)\n"
"    {\n"
"      element_buffer[get_local_id(0)] = elements[i];\n"
"      unsigned int tmp = column_indices[i];\n"
"      col_index_buffer[get_local_id(0)] = tmp;\n"
"      vector_buffer[get_local_id(0)] = vector[tmp];\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    //now a single thread does the remaining work in shared memory:\n"
"    if (get_local_id(0) == 0)\n"
"    {\n"
"      // traverse through all the loaded data from back to front:\n"
"      for (unsigned int k2=0; k2<get_local_size(0); ++k2)\n"
"      {\n"
"        unsigned int k = (get_local_size(0) - k2) - 1;\n"
"        \n"
"        if (i+k >= nnz)\n"
"          continue;\n"
"        \n"
"        if (col_index_buffer[k] > row_at_window_start) //use recently computed results\n"
"          current_vector_entry -= element_buffer[k] * vector_buffer[k];\n"
"        else if (col_index_buffer[k] > current_row) //use buffered data\n"
"          current_vector_entry -= element_buffer[k] * vector[col_index_buffer[k]];\n"
"        \n"
"        if (i+k == next_row) //current row is finished. Write back result\n"
"        {\n"
"          vector[current_row] = current_vector_entry;\n"
"          if (current_row > 0) //load next row's data\n"
"          {\n"
"            --current_row;\n"
"            next_row = row_indices[current_row];\n"
"            current_vector_entry = vector[current_row];\n"
"          }\n"
"        }\n"
"        \n"
"        \n"
"      } // for k\n"
"      \n"
"      row_at_window_start = current_row;\n"
"    } // if (get_local_id(0) == 0)\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    \n"
"    if (i < get_local_size(0))\n"
"      break;\n"
"    \n"
"    i -= get_local_size(0);\n"
"  } //for i\n"
"}\n"
; //compressed_matrix_align1_unit_lu_backward

const char * const compressed_matrix_align1_lu_forward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void lu_forward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int col_index_buffer[128];\n"
"  __local float element_buffer[128];\n"
"  __local float vector_buffer[128];\n"
"          \n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int current_row = 0;\n"
"  unsigned int row_at_window_start = 0;\n"
"  float current_vector_entry = vector[0];\n"
"  float diagonal_entry;\n"
"  unsigned int loop_end = (nnz / get_local_size(0) + 1) * get_local_size(0);\n"
"  unsigned int next_row = row_indices[1];\n"
"  \n"
"  for (unsigned int i = get_local_id(0); i < loop_end; i += get_local_size(0))\n"
"  {\n"
"    //load into shared memory (coalesced access):\n"
"    if (i < nnz)\n"
"    {\n"
"      element_buffer[get_local_id(0)] = elements[i];\n"
"      unsigned int tmp = column_indices[i];\n"
"      col_index_buffer[get_local_id(0)] = tmp;\n"
"      vector_buffer[get_local_id(0)] = vector[tmp];\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    //now a single thread does the remaining work in shared memory:\n"
"    if (get_local_id(0) == 0)\n"
"    {\n"
"      // traverse through all the loaded data:\n"
"      for (unsigned int k=0; k<get_local_size(0); ++k)\n"
"      {\n"
"        if (current_row < size && i+k == next_row) //current row is finished. Write back result\n"
"        {\n"
"          vector[current_row] = current_vector_entry / diagonal_entry;\n"
"          ++current_row;\n"
"          if (current_row < size) //load next row's data\n"
"          {\n"
"            next_row = row_indices[current_row+1];\n"
"            current_vector_entry = vector[current_row];\n"
"          }\n"
"        }\n"
"        \n"
"        if (current_row < size && col_index_buffer[k] < current_row) //substitute\n"
"        {\n"
"          if (col_index_buffer[k] < row_at_window_start) //use recently computed results\n"
"            current_vector_entry -= element_buffer[k] * vector_buffer[k];\n"
"          else if (col_index_buffer[k] < current_row) //use buffered data\n"
"            current_vector_entry -= element_buffer[k] * vector[col_index_buffer[k]];\n"
"        }\n"
"        else if (col_index_buffer[k] == current_row)\n"
"          diagonal_entry = element_buffer[k];\n"
"      } // for k\n"
"      \n"
"      row_at_window_start = current_row;\n"
"    } // if (get_local_id(0) == 0)\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"  } //for i\n"
"}\n"
; //compressed_matrix_align1_lu_forward

const char * const compressed_matrix_align1_trans_unit_lu_backward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void trans_unit_lu_backward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int row_index_lookahead[256];\n"
"  __local unsigned int row_index_buffer[256];\n"
"  \n"
"  unsigned int row_index;\n"
"  unsigned int col_index;\n"
"  float matrix_entry;\n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int row_at_window_start = size;\n"
"  unsigned int row_at_window_end;\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0) + 1) * get_local_size(0);\n"
"  \n"
"  for (unsigned int i2 = get_local_id(0); i2 < loop_end; i2 += get_local_size(0))\n"
"  {\n"
"    unsigned int i = (nnz - i2) - 1;\n"
"    col_index    = (i2 < nnz) ? column_indices[i] : 0;\n"
"    matrix_entry = (i2 < nnz) ? elements[i]       : 0;\n"
"    row_index_lookahead[get_local_id(0)] = (row_at_window_start >= get_local_id(0)) ? row_indices[row_at_window_start - get_local_id(0)] : 0;\n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    if (i2 < nnz)\n"
"    {\n"
"      unsigned int row_index_dec = 0;\n"
"      while (row_index_lookahead[row_index_dec] > i)\n"
"        ++row_index_dec;\n"
"      row_index = row_at_window_start - row_index_dec;\n"
"      row_index_buffer[get_local_id(0)] = row_index;\n"
"    }\n"
"    else\n"
"    {\n"
"      row_index = size+1;\n"
"      row_index_buffer[get_local_id(0)] = 0;\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    row_at_window_start = row_index_buffer[0];\n"
"    row_at_window_end   = row_index_buffer[get_local_size(0) - 1];\n"
"    \n"
"    //backward elimination\n"
"    for (unsigned int row2 = 0; row2 <= (row_at_window_start - row_at_window_end); ++row2) \n"
"    { \n"
"      unsigned int row = row_at_window_start - row2;\n"
"      float result_entry = vector[row];\n"
"      \n"
"      if ( (row_index == row) && (col_index < row) )\n"
"        vector[col_index] -= result_entry * matrix_entry; \n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    }\n"
"    \n"
"    row_at_window_start = row_at_window_end;\n"
"  }\n"
"}\n"
; //compressed_matrix_align1_trans_unit_lu_backward

const char * const compressed_matrix_align1_jacobi = 
"__kernel void jacobi(\n"
" __global const unsigned int * row_indices,\n"
" __global const unsigned int * column_indices,\n"
" __global const float * elements,\n"
" float weight,\n"
" __global const float * old_result,\n"
" __global float * new_result,\n"
" __global const float * rhs,\n"
" unsigned int size)\n"
" {\n"
"  float sum, diag=1;\n"
"  int col;\n"
"  for (unsigned int i = get_global_id(0); i < size; i += get_global_size(0))\n"
"  {\n"
"    sum = 0;\n"
"    for (unsigned int j = row_indices[i]; j<row_indices[i+1]; j++)\n"
"    {\n"
"      col = column_indices[j];\n"
"      if (i == col)\n"
"	diag = elements[j];\n"
"      else \n"
"	sum += elements[j] * old_result[col]; \n"
"    } \n"
"      new_result[i] = weight * (rhs[i]-sum) / diag + (1-weight) * old_result[i]; \n"
"   } \n"
" } \n"
; //compressed_matrix_align1_jacobi

const char * const compressed_matrix_align1_trans_lu_forward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void trans_lu_forward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global const float * diagonal_entries,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int row_index_lookahead[256];\n"
"  __local unsigned int row_index_buffer[256];\n"
"  \n"
"  unsigned int row_index;\n"
"  unsigned int col_index;\n"
"  float matrix_entry;\n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int row_at_window_start = 0;\n"
"  unsigned int row_at_window_end = 0;\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0) + 1) * get_local_size(0);\n"
"  \n"
"  for (unsigned int i = get_local_id(0); i < loop_end; i += get_local_size(0))\n"
"  {\n"
"    col_index    = (i < nnz) ? column_indices[i] : 0;\n"
"    matrix_entry = (i < nnz) ? elements[i]       : 0;\n"
"    row_index_lookahead[get_local_id(0)] = (row_at_window_start + get_local_id(0) < size) ? row_indices[row_at_window_start + get_local_id(0)] : size - 1;\n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    if (i < nnz)\n"
"    {\n"
"      unsigned int row_index_inc = 0;\n"
"      while (i >= row_index_lookahead[row_index_inc + 1])\n"
"        ++row_index_inc;\n"
"      row_index = row_at_window_start + row_index_inc;\n"
"      row_index_buffer[get_local_id(0)] = row_index;\n"
"    }\n"
"    else\n"
"    {\n"
"      row_index = size+1;\n"
"      row_index_buffer[get_local_id(0)] = size - 1;\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    row_at_window_start = row_index_buffer[0];\n"
"    row_at_window_end   = row_index_buffer[get_local_size(0) - 1];\n"
"    \n"
"    //forward elimination\n"
"    for (unsigned int row = row_at_window_start; row <= row_at_window_end; ++row) \n"
"    { \n"
"      float result_entry = vector[row] / diagonal_entries[row];\n"
"      \n"
"      if ( (row_index == row) && (col_index > row) )\n"
"        vector[col_index] -= result_entry * matrix_entry; \n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    }\n"
"    \n"
"    row_at_window_start = row_at_window_end;\n"
"  }\n"
"  \n"
"  // final step: Divide vector by diagonal entries:\n"
"  for (unsigned int i = get_local_id(0); i < size; i += get_local_size(0))\n"
"    vector[i] /= diagonal_entries[i];\n"
"}\n"
; //compressed_matrix_align1_trans_lu_forward

const char * const compressed_matrix_align1_row_info_extractor = 
"__kernel void row_info_extractor(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * result,\n"
"          unsigned int size,\n"
"          unsigned int option\n"
"          ) \n"
"{ \n"
"  for (unsigned int row = get_global_id(0); row < size; row += get_global_size(0))\n"
"  {\n"
"    float value = 0;\n"
"    unsigned int row_end = row_indices[row+1];\n"
"    \n"
"    switch (option)\n"
"    {\n"
"      case 0: //inf-norm\n"
"        for (unsigned int i = row_indices[row]; i < row_end; ++i)\n"
"          value = max(value, fabs(elements[i]));\n"
"        break;\n"
"        \n"
"      case 1: //1-norm\n"
"        for (unsigned int i = row_indices[row]; i < row_end; ++i)\n"
"          value += fabs(elements[i]);\n"
"        break;\n"
"        \n"
"      case 2: //2-norm\n"
"        for (unsigned int i = row_indices[row]; i < row_end; ++i)\n"
"          value += elements[i] * elements[i];\n"
"        value = sqrt(value);\n"
"        break;\n"
"        \n"
"      case 3: //diagonal entry\n"
"        for (unsigned int i = row_indices[row]; i < row_end; ++i)\n"
"        {\n"
"          if (column_indices[i] == row)\n"
"          {\n"
"            value = elements[i];\n"
"            break;\n"
"          }\n"
"        }\n"
"        break;\n"
"        \n"
"      default:\n"
"        break;\n"
"    }\n"
"    result[row] = value;\n"
"  }\n"
"}\n"
; //compressed_matrix_align1_row_info_extractor

const char * const compressed_matrix_align1_vec_mul = 
"__kernel void vec_mul(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global const float * vector,  \n"
"          __global float * result,\n"
"          unsigned int size) \n"
"{ \n"
"  for (unsigned int row = get_global_id(0); row < size; row += get_global_size(0))\n"
"  {\n"
"    float dot_prod = 0.0f;\n"
"    unsigned int row_end = row_indices[row+1];\n"
"    for (unsigned int i = row_indices[row]; i < row_end; ++i)\n"
"      dot_prod += elements[i] * vector[column_indices[i]];\n"
"    result[row] = dot_prod;\n"
"  }\n"
"}\n"
; //compressed_matrix_align1_vec_mul

const char * const compressed_matrix_align1_trans_unit_lu_forward = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void trans_unit_lu_forward(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  __local unsigned int row_index_lookahead[256];\n"
"  __local unsigned int row_index_buffer[256];\n"
"  \n"
"  unsigned int row_index;\n"
"  unsigned int col_index;\n"
"  float matrix_entry;\n"
"  unsigned int nnz = row_indices[size];\n"
"  unsigned int row_at_window_start = 0;\n"
"  unsigned int row_at_window_end = 0;\n"
"  unsigned int loop_end = ( (nnz - 1) / get_local_size(0) + 1) * get_local_size(0);\n"
"  \n"
"  for (unsigned int i = get_local_id(0); i < loop_end; i += get_local_size(0))\n"
"  {\n"
"    col_index    = (i < nnz) ? column_indices[i] : 0;\n"
"    matrix_entry = (i < nnz) ? elements[i]       : 0;\n"
"    row_index_lookahead[get_local_id(0)] = (row_at_window_start + get_local_id(0) < size) ? row_indices[row_at_window_start + get_local_id(0)] : size - 1;\n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    if (i < nnz)\n"
"    {\n"
"      unsigned int row_index_inc = 0;\n"
"      while (i >= row_index_lookahead[row_index_inc + 1])\n"
"        ++row_index_inc;\n"
"      row_index = row_at_window_start + row_index_inc;\n"
"      row_index_buffer[get_local_id(0)] = row_index;\n"
"    }\n"
"    else\n"
"    {\n"
"      row_index = size+1;\n"
"      row_index_buffer[get_local_id(0)] = size - 1;\n"
"    }\n"
"    \n"
"    barrier(CLK_LOCAL_MEM_FENCE);\n"
"    \n"
"    row_at_window_start = row_index_buffer[0];\n"
"    row_at_window_end   = row_index_buffer[get_local_size(0) - 1];\n"
"    \n"
"    //forward elimination\n"
"    for (unsigned int row = row_at_window_start; row <= row_at_window_end; ++row) \n"
"    { \n"
"      float result_entry = vector[row];\n"
"      \n"
"      if ( (row_index == row) && (col_index > row) )\n"
"        vector[col_index] -= result_entry * matrix_entry; \n"
"      barrier(CLK_GLOBAL_MEM_FENCE);\n"
"    }\n"
"    \n"
"    row_at_window_start = row_at_window_end;\n"
"  }\n"
"}\n"
; //compressed_matrix_align1_trans_unit_lu_forward

const char * const compressed_matrix_align1_block_trans_lu_backward = 
" __kernel void block_trans_lu_backward(\n"
"           __global const unsigned int * row_jumper_U,      //U part (note that U is transposed in memory)\n"
"           __global const unsigned int * column_indices_U,\n"
"           __global const float * elements_U,\n"
"           __global const float * diagonal_U,\n"
"           __global const unsigned int * block_offsets,\n"
"           __global float * result,\n"
"           unsigned int size)\n"
" {\n"
"   unsigned int col_start = block_offsets[2*get_group_id(0)];\n"
"   unsigned int col_stop  = block_offsets[2*get_group_id(0)+1];\n"
"   unsigned int row_start;\n"
"   unsigned int row_stop;\n"
"   float result_entry = 0;\n"
"   if (col_start >= col_stop)\n"
"     return;\n"
"   //backward elimination, using U and diagonal_U\n"
"   for (unsigned int iter = 0; iter < col_stop - col_start; ++iter) \n"
"   { \n"
"     unsigned int col = (col_stop - iter) - 1;\n"
"     result_entry = result[col] / diagonal_U[col];\n"
"     row_start = row_jumper_U[col]; \n"
"     row_stop  = row_jumper_U[col + 1]; \n"
"     for (unsigned int buffer_index = row_start + get_local_id(0); buffer_index < row_stop; buffer_index += get_local_size(0)) \n"
"       result[column_indices_U[buffer_index]] -= result_entry * elements_U[buffer_index];\n"
"     barrier(CLK_GLOBAL_MEM_FENCE); \n"
"   } \n"
"   //divide result vector by diagonal:\n"
"   for (unsigned int col = col_start + get_local_id(0); col < col_stop; col += get_local_size(0)) \n"
"     result[col] /= diagonal_U[col];\n"
" };\n"
; //compressed_matrix_align1_block_trans_lu_backward

const char * const compressed_matrix_align1_trans_unit_lu_forward_slow = 
" \n"
"// compute y in Ly = z for incomplete LU factorizations of a sparse matrix in compressed format\n"
"__kernel void trans_unit_lu_forward_slow(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const unsigned int * column_indices, \n"
"          __global const float * elements,\n"
"          __global float * vector,\n"
"          unsigned int size) \n"
"{\n"
"  for (unsigned int row = 0; row < size; ++row) \n"
"  { \n"
"    float result_entry = vector[row]; \n"
"    \n"
"    unsigned int row_start = row_indices[row]; \n"
"    unsigned int row_stop  = row_indices[row + 1];\n"
"    for (unsigned int entry_index = row_start + get_local_id(0); entry_index < row_stop; entry_index += get_local_size(0)) \n"
"    {\n"
"      unsigned int col_index = column_indices[entry_index];\n"
"      if (col_index > row)\n"
"        vector[col_index] -= result_entry * elements[entry_index]; \n"
"    }\n"
"    \n"
"    barrier(CLK_GLOBAL_MEM_FENCE);\n"
"  } \n"
"}\n"
; //compressed_matrix_align1_trans_unit_lu_forward_slow

const char * const compressed_matrix_align1_block_trans_unit_lu_forward = 
" __kernel void block_trans_unit_lu_forward(\n"
"           __global const unsigned int * row_jumper_L,      //L part (note that L is transposed in memory)\n"
"           __global const unsigned int * column_indices_L, \n"
"           __global const float * elements_L,\n"
"           __global const unsigned int * block_offsets,\n"
"           __global float * result,\n"
"           unsigned int size)\n"
" {\n"
"   unsigned int col_start = block_offsets[2*get_group_id(0)];\n"
"   unsigned int col_stop  = block_offsets[2*get_group_id(0)+1];\n"
"   unsigned int row_start = row_jumper_L[col_start];\n"
"   unsigned int row_stop;\n"
"   float result_entry = 0;\n"
"   if (col_start >= col_stop)\n"
"     return;\n"
"   //forward elimination, using L:\n"
"   for (unsigned int col = col_start; col < col_stop; ++col)\n"
"   {\n"
"     result_entry = result[col];\n"
"     row_stop = row_jumper_L[col + 1];\n"
"     for (unsigned int buffer_index = row_start + get_local_id(0); buffer_index < row_stop; buffer_index += get_local_size(0))\n"
"       result[column_indices_L[buffer_index]] -= result_entry * elements_L[buffer_index]; \n"
"     row_start = row_stop; //for next iteration (avoid unnecessary loads from GPU RAM)\n"
"     barrier(CLK_GLOBAL_MEM_FENCE);\n"
"   } \n"
" };\n"
; //compressed_matrix_align1_block_trans_unit_lu_forward

const char * const compressed_matrix_align4_vec_mul = 
"__kernel void vec_mul(\n"
"          __global const unsigned int * row_indices,\n"
"          __global const uint4 * column_indices, \n"
"          __global const float4 * elements,\n"
"          __global const float * vector,  \n"
"          __global float * result,\n"
"          unsigned int size)\n"
"{ \n"
"  float dot_prod;\n"
"  unsigned int start, next_stop;\n"
"  uint4 col_idx;\n"
"  float4 tmp_vec;\n"
"  float4 tmp_entries;\n"
"  for (unsigned int row = get_global_id(0); row < size; row += get_global_size(0))\n"
"  {\n"
"    dot_prod = 0.0f;\n"
"    start = row_indices[row] / 4;\n"
"    next_stop = row_indices[row+1] / 4;\n"
"    for (unsigned int i = start; i < next_stop; ++i)\n"
"    {\n"
"      col_idx = column_indices[i];\n"
"      tmp_entries = elements[i];\n"
"      tmp_vec.x = vector[col_idx.x];\n"
"      tmp_vec.y = vector[col_idx.y];\n"
"      tmp_vec.z = vector[col_idx.z];\n"
"      tmp_vec.w = vector[col_idx.w];\n"
"      dot_prod += dot(tmp_entries, tmp_vec);\n"
"    }\n"
"    result[row] = dot_prod;\n"
"  }\n"
"}\n"
; //compressed_matrix_align4_vec_mul

  }  //namespace kernels
 }  //namespace linalg
}  //namespace viennacl
#endif

